diff --git a/.config/e2e-distributed-selection.txt b/.config/e2e-distributed-selection.txt index 6cc442542..de2cdc658 100644 --- a/.config/e2e-distributed-selection.txt +++ b/.config/e2e-distributed-selection.txt @@ -1,2 +1,2 @@ -sha256-linux=4a399af73f3a954702967b80649958ae127f4e14560194e193de37e3eac5a17a -sha256-darwin=4a399af73f3a954702967b80649958ae127f4e14560194e193de37e3eac5a17a +sha256-linux=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e +sha256-darwin=775825dcb2b4997c4fa24bd9ba9c0546316c503f4d5e369c39abff0678c95e8e diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index 349e17819..1ab608350 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3 -sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87 +sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5 +sha256-linux=9351e25b45bf7dfce18b951a5e3740225f457cacc53b8bf9f500f6947763ec0e diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index 1bb5125c9..bec86f799 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=364f2329a7b72eb9f1608dbe1a3af37af4095354014f3cbe23ca448492d89961 -sha256-linux=60983f1ebe7068cf660d473c5f76c76a650410ccc99d71934ddca7fd67607987 +sha256-darwin=83a7dcaffd5a789517ae9f02a224f66a9713937885cff96fca2ad7e216f197ae +sha256-linux=626c10f8c964507ff987b6c86069e9019dc6d2ae7fb02db9be5df5aa8cc5145b diff --git a/.config/make/tests.mak b/.config/make/tests.mak index d9c34e347..483ae9c4a 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -32,6 +32,16 @@ script-tests: ## Run shell script tests ./scripts/test_hotpath_warp_ab_gate.sh ./scripts/test_hotpath_warp_abba.sh ./scripts/test_scanner_validation_harness.sh + ./scripts/test_scanner_heal_checkpoint_crash_evidence.sh + ./scripts/test_scanner_heal_authority_evidence.sh + ./scripts/test_scanner_heal_scoped_ack_evidence.sh + ./scripts/test_scanner_heal_legacy_rollback_evidence.sh + ./scripts/test_scanner_heal_g14_multiset_evidence.sh + ./scripts/test_scanner_heal_scheduler_pressure_evidence.sh + ./scripts/test_scanner_heal_status_outcome_evidence.sh + ./scripts/test_scanner_heal_maintenance_evidence.sh + ./scripts/test_scanner_heal_w13_mrf_evidence.sh + ./scripts/test_scanner_heal_w16_recovery_evidence.sh ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh diff --git a/.config/scanner-heal-required-tests.json b/.config/scanner-heal-required-tests.json index b7fb4b268..ff24d4a92 100644 --- a/.config/scanner-heal-required-tests.json +++ b/.config/scanner-heal-required-tests.json @@ -1,5 +1,5 @@ { - "schema": 1, + "schema": 2, "cases": { "background-target-restart": { "gate": "G14", @@ -13,6 +13,7 @@ "min_objects": 9, "max_objects": 65, "topology": {"nodes": 4, "drives_per_node": 1}, + "erasure": {"data_blocks": 2, "parity_blocks": 2}, "scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4." }, "background-target-crash": { @@ -27,30 +28,422 @@ "min_objects": 9, "max_objects": 65, "topology": {"nodes": 4, "drives_per_node": 1}, + "erasure": {"data_blocks": 2, "parity_blocks": 2}, "scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4." + }, + "ec84-target-drive-restart": { + "gate": "G14", + "task": "W20/W21", + "lane": "e2e-distributed", + "suite": "e2e_test", + "name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart", + "oracle": "ec84-target-drive-restart.json", + "evidence": "process-restart", + "unclean_shutdown_marker": false, + "min_objects": 5, + "max_objects": 5, + "topology": {"nodes": 3, "drives_per_node": 4}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA." + }, + "background-target-restart-ec8-4": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart", + "oracle": "background-target-restart-ec8-4.json", + "evidence": "process-restart", + "unclean_shutdown_marker": false, + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 3, "drives_per_node": 4}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "scope": "Target process restart during partial background rebuild on a single 3x4 EC8+4 set; exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool." + }, + "background-target-crash-ec8-4": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash", + "oracle": "background-target-crash-ec8-4.json", + "evidence": "process-crash-restart", + "unclean_shutdown_marker": true, + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 3, "drives_per_node": 4}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool." + }, + "background-target-restart-ec8-4-multi-set": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart", + "oracle": "background-target-restart-ec8-4-multi-set.json", + "evidence": "process-restart", + "unclean_shutdown_marker": false, + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 3, "drives_per_node": 8}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "sets": 2, + "pools": 1, + "scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool." + }, + "background-target-crash-ec8-4-multi-pool": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash", + "oracle": "background-target-crash-ec8-4-multi-pool.json", + "evidence": "process-crash-restart", + "unclean_shutdown_marker": true, + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 3, "drives_per_node": 12}, + "erasure": {"data_blocks": 8, "parity_blocks": 4}, + "erasure_set_drive_count": 12, + "sets": 3, + "pools": 3, + "outage_target_manifest_required": false, + "scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with down-window outage PUT refusal recorded and a deferred post-rejoin outage object verified through S3 when the full target pool was offline." } }, - "release_pending": { - "G01": "W02/W04 complete root and quota authority coverage", - "G02": "W03 bounded checkpoint progress and independent version inventory", - "G03": "W17/W18 exact scoped ACK with durable publication and mixed peers", - "G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary", - "G05": "W06/W07 per-object outcomes and bounded terminal retention", - "G06": "W06/W08/W23 concurrent status, legacy clients and truncation", - "G07": "W12/W13/W14 durable MRF responsibility at every commit boundary", - "G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix", - "G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads", - "G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery", - "G11": "W04/W19/W24 maintenance and complete producer coverage", - "G12": "W02/W15/W16 both quota paths during reset and settlement", - "G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail", - "G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage", - "P1": "W20 measured cold-walk share and foreground latency/throughput", - "P2": "W20/W24 measured post-stop convergence and cold segment reuse", - "P3": "W20 measured two-hour pressure/heal capacity and recovery window", - "P4": "W20 measured MRF scale and replay cost with retained responsibility", - "R-E": "W03/W05 fixed-budget real process restart through enumeration and classification", - "R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace", - "R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement" - } + "release_lanes": { + "single-set-restart": { + "status": "implemented", + "cases": ["background-target-restart", "background-target-crash"], + "covers": ["four-node one-drive topology", "unversioned objects", "target restart/crash"] + }, + "authority-coverage": { + "status": "pending", + "gates": ["G01", "G12"], + "requires": ["root authority coverage", "quota authority coverage"] + }, + "checkpoint-and-crash": { + "status": "pending", + "gates": ["G02", "G04", "R-E"], + "requires": ["bounded checkpoint progress", "boundary crash matrix", "fixed-budget restart evidence"] + }, + "status-and-outcome": { + "status": "pending", + "gates": ["G05", "G06", "R-D"], + "requires": ["per-object outcomes", "legacy status clients", "manager/event/ledger disposition"] + }, + "mrf-responsibility": { + "status": "pending", + "gates": ["G07", "G08", "P4"], + "requires": ["durable MRF responsibility", "disk-full and replica-loss matrix", "MRF replay cost"] + }, + "mixed-version-rollback": { + "status": "pending", + "gates": ["G03", "G09", "R-L"], + "requires": ["mixed-version peers", "rollback payloads", "crash-safe source retirement"] + }, + "scheduler-pressure": { + "status": "pending", + "gates": ["G10", "P1", "P2", "P3"], + "requires": ["bounded scheduling", "foreground latency and throughput", "two-hour pressure evidence"] + }, + "maintenance-producers": { + "status": "pending", + "gates": ["G11", "G13"], + "requires": ["complete producer coverage", "quorum-minus-one and remount matrix"] + }, + "ec8-4-multiset": { + "status": "pending", + "gates": ["G14"], + "requires": ["3x4 EC8+4 topology", "multi-set coverage", "multi-pool coverage"] + } + }, + "release_requirements": [ + { + "gate": "G01", + "task": "W02/W04", + "lane": "authority-coverage", + "status": "pending", + "description": "Complete root and quota authority coverage", + "requires": ["root authority evidence", "quota authority evidence"], + "evidence_fields": [ + "root_authority_evidence", + "quota_authority_evidence" + ] + }, + { + "gate": "G02", + "task": "W03", + "lane": "checkpoint-and-crash", + "status": "pending", + "description": "Bounded checkpoint progress and independent version inventory", + "requires": ["bounded checkpoint oracle", "independent version inventory"], + "evidence_fields": [ + "bounded_checkpoint_oracle", + "independent_version_inventory" + ] + }, + { + "gate": "G03", + "task": "W17/W18", + "lane": "mixed-version-rollback", + "status": "pending", + "description": "Exact scoped ACK with durable publication and mixed peers", + "requires": ["durable scoped ACK publication", "mixed-peer evidence"], + "evidence_fields": [ + "durable_root_publication_proof", + "scoped_ack_request_identity", + "participating_peer_capability_snapshot", + "mixed_peer_ack_fallback_oracle" + ] + }, + { + "gate": "G04", + "task": "W03/W15/W16", + "lane": "checkpoint-and-crash", + "status": "pending", + "description": "Crash at every cache, root, floor and intent boundary", + "requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"], + "evidence_fields": [ + "cache_boundary_crash_evidence", + "root_floor_intent_crash_evidence" + ] + }, + { + "gate": "G05", + "task": "W06/W07", + "lane": "status-and-outcome", + "status": "pending", + "description": "Per-object outcomes and bounded terminal retention", + "requires": ["per-object outcome oracle", "terminal retention bounds"], + "evidence_fields": [ + "per_object_outcome_oracle", + "terminal_retention_bounds" + ] + }, + { + "gate": "G06", + "task": "W06/W08/W23", + "lane": "status-and-outcome", + "status": "pending", + "description": "Concurrent status, legacy clients and truncation", + "requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"], + "evidence_fields": [ + "concurrent_status_evidence", + "legacy_client_compatibility", + "truncation_behavior" + ] + }, + { + "gate": "G07", + "task": "W12/W13/W14", + "lane": "mrf-responsibility", + "status": "pending", + "description": "Durable MRF responsibility at every commit boundary", + "requires": ["MRF responsibility oracle", "commit-boundary crash matrix"], + "evidence_fields": [ + "mrf_responsibility_oracle", + "commit_boundary_crash_matrix" + ] + }, + { + "gate": "G08", + "task": "W12/W13/W14", + "lane": "mrf-responsibility", + "status": "pending", + "description": "MRF capacity, disk-full and replica-loss matrix", + "requires": ["MRF capacity evidence", "disk-full matrix", "replica-loss matrix"], + "evidence_fields": [ + "mrf_capacity_evidence", + "disk_full_matrix", + "replica_loss_matrix" + ] + }, + { + "gate": "G09", + "task": "W13/W18/W23", + "lane": "mixed-version-rollback", + "status": "pending", + "description": "Actual mixed-version reader/writer and rollback payloads", + "requires": ["mixed-version reader evidence", "mixed-version writer evidence", "rollback payload evidence"], + "evidence_fields": [ + "mixed_version_reader_evidence", + "mixed_version_writer_evidence", + "rollback_payload_evidence" + ] + }, + { + "gate": "G10", + "task": "W05/W09/W10/W11", + "lane": "scheduler-pressure", + "status": "pending", + "description": "Bounded scheduling and pressure recovery", + "requires": ["scheduler bound evidence", "pressure recovery evidence"], + "evidence_fields": [ + "scheduler_bound_evidence", + "pressure_recovery_evidence" + ] + }, + { + "gate": "G11", + "task": "W04/W19/W24", + "lane": "maintenance-producers", + "status": "pending", + "description": "Maintenance and complete producer coverage", + "requires": [ + "maintenance producer matrix", + "complete producer inventory", + "segment activation preflight" + ], + "evidence_fields": [ + "maintenance_producer_matrix", + "complete_producer_inventory", + "segment_activation_preflight" + ] + }, + { + "gate": "G12", + "task": "W02/W15/W16", + "lane": "authority-coverage", + "status": "pending", + "description": "Both quota paths during reset and settlement", + "requires": ["reset quota-path evidence", "settlement quota-path evidence"], + "evidence_fields": [ + "reset_quota_path_evidence", + "settlement_quota_path_evidence" + ] + }, + { + "gate": "G13", + "task": "W07/W14", + "lane": "maintenance-producers", + "status": "pending", + "description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail", + "requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"], + "evidence_fields": [ + "quorum_minus_one_matrix", + "unknown_disk_remount_matrix", + "object_lock_dry_run_grace_evidence" + ] + }, + { + "gate": "G14", + "task": "W20/W21", + "lane": "ec8-4-multiset", + "status": "pending", + "description": "Same-window field evidence with 3x4 EC8+4 and multi-set/pool coverage", + "requires": [ + "same-window field evidence", + "3x4 EC8+4 evidence", + "multi-set evidence", + "multi-pool evidence", + "distributed segment invalidation evidence" + ], + "evidence_fields": [ + "same_window_field_evidence", + "ec8_4_evidence", + "multi_set_evidence", + "multi_pool_evidence", + "distributed_segment_invalidation_evidence" + ] + }, + { + "gate": "P1", + "task": "W20", + "lane": "scheduler-pressure", + "status": "pending", + "description": "Measured cold-walk share and foreground latency/throughput", + "requires": ["cold-walk share measurement", "foreground latency/throughput measurement"], + "evidence_fields": [ + "cold_walk_share_measurement", + "foreground_latency_throughput_measurement", + "profile_evidence" + ] + }, + { + "gate": "P2", + "task": "W20/W24", + "lane": "scheduler-pressure", + "status": "pending", + "description": "Measured post-stop convergence and cold segment reuse", + "requires": ["post-stop convergence measurement", "cold segment reuse measurement"], + "evidence_fields": [ + "post_stop_convergence_measurement", + "cold_segment_reuse_measurement" + ] + }, + { + "gate": "P3", + "task": "W20", + "lane": "scheduler-pressure", + "status": "pending", + "description": "Measured two-hour pressure/heal capacity and recovery window", + "requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"], + "evidence_fields": [ + "two_hour_pressure_measurement", + "heal_capacity_measurement", + "recovery_window_measurement" + ] + }, + { + "gate": "P4", + "task": "W20", + "lane": "mrf-responsibility", + "status": "pending", + "description": "Measured MRF scale and replay cost with retained responsibility", + "requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence", "cleanup/GC soak evidence"], + "evidence_fields": [ + "mrf_scale_measurement", + "mrf_replay_cost_measurement", + "retained_responsibility_evidence", + "mrf_cleanup_gc_soak_evidence" + ] + }, + { + "gate": "R-E", + "task": "W03/W05", + "lane": "checkpoint-and-crash", + "status": "pending", + "description": "Fixed-budget real process restart through enumeration and classification", + "requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"], + "evidence_fields": [ + "fixed_budget_restart_evidence", + "enumeration_evidence", + "classification_evidence" + ] + }, + { + "gate": "R-D", + "task": "W07/W14", + "lane": "status-and-outcome", + "status": "pending", + "description": "Manager-to-event-to-ledger exact disposition, including grace", + "requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"], + "evidence_fields": [ + "manager_disposition_evidence", + "event_disposition_evidence", + "ledger_disposition_evidence", + "grace_handling" + ] + }, + { + "gate": "R-L", + "task": "W13/W14", + "lane": "mixed-version-rollback", + "status": "pending", + "description": "Legacy source conflicts, migration gaps and crash-safe source retirement", + "requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"], + "evidence_fields": [ + "legacy_source_conflict_evidence", + "migration_gap_evidence", + "crash_safe_source_retirement_evidence" + ] + } + ] } diff --git a/.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml b/.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml index ecaf98874..7657e60b0 100644 --- a/.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml +++ b/.docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml @@ -43,6 +43,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger # should show richer nested spans during request-path verification. - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} @@ -89,6 +90,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger # should show richer nested spans during request-path verification. - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} @@ -135,6 +137,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger # should show richer nested spans during request-path verification. - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} @@ -181,6 +184,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} # `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger # should show richer nested spans during request-path verification. - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} diff --git a/.docker/compose/docker-compose.cluster.local-build.yml b/.docker/compose/docker-compose.cluster.local-build.yml index 80af50c2b..76dcce105 100644 --- a/.docker/compose/docker-compose.cluster.local-build.yml +++ b/.docker/compose/docker-compose.cluster.local-build.yml @@ -26,6 +26,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false} - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false} @@ -72,6 +73,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false} - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false} @@ -118,6 +120,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false} - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false} @@ -164,6 +167,7 @@ services: - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local} - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local} - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} - RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false} - RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false} diff --git a/.docker/compose/docker-compose.observability.yaml b/.docker/compose/docker-compose.observability.yaml index 03b4d765d..a52065ce6 100644 --- a/.docker/compose/docker-compose.observability.yaml +++ b/.docker/compose/docker-compose.observability.yaml @@ -201,6 +201,7 @@ services: - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=debug platform: linux/amd64 ports: @@ -220,6 +221,7 @@ services: - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=debug platform: linux/amd64 ports: @@ -239,6 +241,7 @@ services: - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=debug platform: linux/amd64 ports: @@ -258,6 +261,7 @@ services: - RUSTFS_ADDRESS=:9000 - RUSTFS_CONSOLE_ENABLE=true - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_LOGGER_LEVEL=debug platform: linux/amd64 ports: diff --git a/.docker/observability/README.md b/.docker/observability/README.md index 8781c1b4c..d9b61b521 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -47,6 +47,10 @@ Three pre-built Grafana dashboards are included for monitoring RustFS GET perfor | **GET Resource Impact** | `grafana-get-resource-impact.json` | Monitors resource usage: concurrent requests, IO queue utilization, disk permit wait, RSS trend | | **Object Data Cache** | `grafana-object-data-cache.json` | Monitors the GET body cache (`rustfs_object_data_cache_*`): hit ratio, lookup/plan/fill outcomes, fill duration quantiles, hit vs fill throughput, entries/weighted bytes, inflight fills, memory-pressure skips, invalidations, and size-class breakdowns | +### Storage Metrics + +Storage panels require `prometheus-rules/rustfs-storage.yml` and the cluster resource attribute. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for ownership, observer selection, freshness, and rolling upgrades. + ### Prometheus Alert Rules The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-configured alerting rules: diff --git a/.docker/observability/docker-compose-example-for-rustfs.yml b/.docker/observability/docker-compose-example-for-rustfs.yml index 1d8ce46bb..f2b92c96c 100644 --- a/.docker/observability/docker-compose-example-for-rustfs.yml +++ b/.docker/observability/docker-compose-example-for-rustfs.yml @@ -32,6 +32,7 @@ services: - RUSTFS_SECRET_KEY=rustfsadmin - RUSTFS_OBS_LOGGER_LEVEL=info - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} - RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040 volumes: - rustfs-data:/data/rustfs diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index 5a72306c3..15fbdc34b 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -223,14 +223,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_buckets_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Buckets", - "type": "stat" + "type": "stat", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -289,14 +290,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_objects_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Objects", - "type": "stat" + "type": "stat", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -427,7 +429,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "Used", "range": true, "refId": "A" @@ -438,7 +440,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "hide": false, "legendFormat": "Total", "range": true, @@ -450,7 +452,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "hide": false, "instant": false, "legendFormat": "Percent", @@ -459,7 +461,8 @@ } ], "title": "Capacity", - "type": "stat" + "type": "stat", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -525,7 +528,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_stale_drives{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_stale_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "Stale Drives", "range": true, "refId": "A" @@ -536,14 +539,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_missing_drives{job=~\"$job\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_missing_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "Missing Drives", "range": true, "refId": "B" } ], "title": "Capacity Observation", - "type": "stat" + "type": "stat", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -1989,8 +1993,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"})", - "legendFormat": "{{drive}} (bytes)", + "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}", + "legendFormat": "{{server}} | {{drive}} (bytes)", "range": true, "refId": "A" }, @@ -2000,16 +2004,17 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_system_drive_total_bytes{job=~\"$job\", drive=~\"$drive\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}", "hide": false, "instant": false, - "legendFormat": "{{drive}} (percent)", + "legendFormat": "{{server}} | {{drive}} (percent)", "range": true, "refId": "B" } ], "title": "System Drive Usage", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -2097,14 +2102,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (drive) (rustfs_system_drive_capacity_observation_age_seconds{job=~\"$job\", drive=~\"$drive\"})", - "legendFormat": "{{drive}}", + "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_age_seconds\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}", + "legendFormat": "{{server}} | {{drive}}", "range": true, "refId": "A" } ], "title": "Drive Capacity Observation Age", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -2190,8 +2196,8 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"stale\"})", - "legendFormat": "{{drive}} stale", + "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"stale\"}", + "legendFormat": "{{server}} | {{drive}} stale", "range": true, "refId": "A" }, @@ -2201,14 +2207,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"missing\"})", - "legendFormat": "{{drive}} missing", + "expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"missing\"}", + "legendFormat": "{{server}} | {{drive}} missing", "range": true, "refId": "B" } ], "title": "Drive Capacity Observation State", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -4551,7 +4558,7 @@ "index": 0, "text": "INACTIVE" }, - "to": 1e-9 + "to": 1e-09 }, "type": "range" } @@ -6563,7 +6570,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rustfs_cluster_health_drives_online_count{job=~\"$job\"}", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_online_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "online - {{job}}", "range": true, "refId": "A" @@ -6574,7 +6581,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rustfs_cluster_health_drives_offline_count{job=~\"$job\"}", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_offline_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "offline - {{job}}", "range": true, "refId": "B" @@ -6585,14 +6592,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rustfs_cluster_health_drives_count{job=~\"$job\"}", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", "legendFormat": "total - {{job}}", "range": true, "refId": "C" } ], "title": "Cluster Drive Health Counts", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "collapsed": false, @@ -8554,14 +8562,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "{__name__=~\"rustfs_system_drive_.*\",job=~\"$job\",drive=~\"$drive\"}", - "legendFormat": "{{__name__}} | {{drive}}", + "expr": "rustfs:storage:current{source_metric=~\"rustfs_system_drive_.*\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}", + "legendFormat": "{{server}} | {{source_metric}} | {{drive}}", "range": true, "refId": "A" } ], "title": "System Drive (All)", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -8942,14 +8951,15 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "{__name__=~\"rustfs_cluster_erasure_set_.*\",job=~\"$job\"}", - "legendFormat": "{{__name__}}", + "expr": "rustfs:storage:current{source_metric=~\"rustfs_cluster_erasure_set_.*\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}", + "legendFormat": "{{source_metric}}", "range": true, "refId": "A" } ], "title": "Cluster Erasure Set (All)", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -11716,7 +11726,7 @@ "editorMode": "code", "range": true, "refId": "A", - "expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}", "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}" }, { @@ -11727,12 +11737,13 @@ "editorMode": "code", "range": true, "refId": "B", - "expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})", + "expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_offline_duration_seconds\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}", "legendFormat": "{{server}} | {{drive}} | offline seconds" } ], - "title": "Drive Runtime State and Offline Duration", - "type": "timeseries" + "title": "Observed Cluster Drive State and Offline Duration", + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -11825,12 +11836,13 @@ "editorMode": "code", "range": true, "refId": "A", - "expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))", + "expr": "rate(rustfs_system_drive_api_calls_total{collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]) and ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}", "legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}" } ], "title": "Drive API Calls by Operation", - "type": "timeseries" + "type": "timeseries", + "description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed." }, { "datasource": { @@ -12376,6 +12388,50 @@ "sort": 1, "type": "query" }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)", + "includeAll": false, + "label": "Storage cluster", + "multi": false, + "name": "storage_cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)", + "refId": "PrometheusVariableQueryEditor-storage_cluster" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})", + "includeAll": false, + "label": "Cluster observer", + "multi": false, + "name": "storage_observer", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})", + "refId": "PrometheusVariableQueryEditor-storage_observer" + }, + "refresh": 2, + "regex": "/observer=\"([^\"]+)\"/", + "sort": 1, + "type": "query" + }, { "allValue": ".*", "current": { @@ -12386,7 +12442,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(rustfs_system_drive_api_calls_total,api)", + "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)", "includeAll": true, "label": "Drive API", "multi": true, @@ -12394,7 +12450,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(rustfs_system_drive_api_calls_total,api)", + "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)", "refId": "PrometheusVariableQueryEditor-drive_api" }, "refresh": 2, @@ -12455,7 +12511,7 @@ "text": "All", "value": "$__all" }, - "definition": "label_values(rustfs_system_drive_used_bytes,drive)", + "definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)", "includeAll": true, "label": "Drive", "multi": true, @@ -12463,7 +12519,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(rustfs_system_drive_used_bytes,drive)", + "query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, diff --git a/.docker/observability/prometheus-rules/rustfs-storage.yml b/.docker/observability/prometheus-rules/rustfs-storage.yml new file mode 100644 index 000000000..45f5d956c --- /dev/null +++ b/.docker/observability/prometheus-rules/rustfs-storage.yml @@ -0,0 +1,575 @@ +# Preserve OTLP timestamps in the Collector. Apply timestamp() directly to +# each raw selector, before label rewriting; functions such as label_replace() +# replace the evaluation timestamp and would make a cached value look new. +# Compare publication times at Prometheus millisecond precision so points +# published and exported within the same millisecond are not withheld. +groups: + - name: rustfs-storage-snapshots + interval: 15s + rules: + - record: rustfs:storage_snapshot:fresh + expr: | + rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"} + and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) <= rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) + and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) >= 0) + and (rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"} > 0) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_buckets_total + expr: | + rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_capacity_free_bytes + expr: | + rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_capacity_missing_drives + expr: | + rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_capacity_raw_total_bytes + expr: | + rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_capacity_stale_drives + expr: | + rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_capacity_usable_total_bytes + expr: | + rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_capacity_used_bytes + expr: | + rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_capacity_observation_age_seconds + expr: | + rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_capacity_observation_state + expr: | + rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_free_bytes + expr: | + rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_offline_duration_seconds + expr: | + rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_present + expr: | + rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_runtime_state + expr: | + rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_total_bytes + expr: | + rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_drive_used_bytes + expr: | + rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_data_shards + expr: | + rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_healing_drives_count + expr: | + rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_health + expr: | + rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_online_drives_count + expr: | + rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_overall_health + expr: | + rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_overall_write_quorum + expr: | + rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_parity + expr: | + rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_read_health + expr: | + rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_read_quorum + expr: | + rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_read_tolerance + expr: | + rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_size + expr: | + rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_write_health + expr: | + rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_write_quorum + expr: | + rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_erasure_set_write_tolerance + expr: | + rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_health_drives_count + expr: | + rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_health_drives_offline_count + expr: | + rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_health_drives_online_count + expr: | + rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_cluster_objects_total + expr: | + rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"} + and (timestamp(rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_node_disk_free_bytes + expr: | + rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_node_disk_total_bytes + expr: | + rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_node_disk_used_bytes + expr: | + rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_api_calls_total + expr: | + rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_api_latency_by_api_micros + expr: | + rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_api_latency_micros + expr: | + rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_availability_errors_total + expr: | + rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_capacity_observation_age_seconds + expr: | + rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_capacity_observation_state + expr: | + rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_count + expr: | + rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_deletes_total + expr: | + rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_free_bytes + expr: | + rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_free_inodes + expr: | + rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_healing + expr: | + rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_health + expr: | + rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_info + expr: | + rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_io_errors_total + expr: | + rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_offline_count + expr: | + rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_offline_duration_seconds + expr: | + rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_online_count + expr: | + rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_perc_util + expr: | + rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_present + expr: | + rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_reads_await + expr: | + rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_reads_kb_per_sec + expr: | + rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_reads_per_sec + expr: | + rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_runtime_state + expr: | + rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_scanning + expr: | + rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_timeout_errors_total + expr: | + rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_total_bytes + expr: | + rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_total_inodes + expr: | + rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_used_bytes + expr: | + rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_used_inodes + expr: | + rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_waiting_io + expr: | + rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_writes_await + expr: | + rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_writes_kb_per_sec + expr: | + rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_writes_per_sec + expr: | + rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) + - record: rustfs:storage:current + labels: + source_metric: rustfs_system_drive_writes_total + expr: | + rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"} + and (timestamp(rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"}) + >= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left() + (floor(rustfs:storage_snapshot:fresh * 1000) / 1000)) diff --git a/.docker/observability/tests/storage-rules.test.yml b/.docker/observability/tests/storage-rules.test.yml new file mode 100644 index 000000000..d70f68087 --- /dev/null +++ b/.docker/observability/tests/storage-rules.test.yml @@ -0,0 +1,189 @@ +rule_files: +- ../prometheus-rules/rustfs-storage.yml +evaluation_interval: 15s +tests: +- name: owners, pools, clusters and observer views remain distinct in mixed versions + interval: 1m + input_series: + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"} + values: 100x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"} + values: 100x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"} + values: 100x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"} + values: 100x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"} + values: 1000x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"} + values: 1000x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"} + values: 1000x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"} + values: 1000x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"} + values: 400x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"} + values: 300x8 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="old",server="n0",drive="/data",job="rustfs",instance="collector"} + values: 999x8 + promql_expr_test: + - expr: sum by (rustfs_cluster_id) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",collection_scope="local"}) + eval_time: 2m + exp_samples: + - labels: '{rustfs_cluster_id="a"}' + value: 400 + - labels: '{rustfs_cluster_id="b"}' + value: 4000 + - expr: count by (rustfs_cluster_id,pool_index) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"}) + eval_time: 2m + exp_samples: + - labels: '{rustfs_cluster_id="a",pool_index="0"}' + value: 2 + - labels: '{rustfs_cluster_id="a",pool_index="1"}' + value: 2 + - labels: '{rustfs_cluster_id="b",pool_index="0"}' + value: 2 + - labels: '{rustfs_cluster_id="b",pool_index="1"}' + value: 2 + - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n0"}) + eval_time: 2m + exp_samples: + - labels: '{}' + value: 400 + - expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n1"}) + eval_time: 2m + exp_samples: + - labels: '{}' + value: 300 +- name: removed values and identities cannot rejoin new snapshots; stalled sources expire + interval: 1m + input_series: + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 0 60 _ _ _ _ _ _ _ + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="old"} + values: 1 1 _ _ _ _ _ _ _ + - series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="new"} + values: _ _ 1 1 1 1 1 1 1 + - series: rustfs_system_drive_waiting_io{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data"} + values: 7 7 _ _ _ _ _ _ _ + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data"} + values: 100 100 _ _ _ _ _ _ _ + promql_expr_test: + - expr: rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="old"} + eval_time: 2m + exp_samples: [] + - expr: rustfs:storage:current{source_metric="rustfs_system_drive_waiting_io"} + eval_time: 2m + exp_samples: [] + - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="new"}) + eval_time: 2m + exp_samples: + - labels: '{}' + value: 1 + - expr: rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"} + eval_time: 5m + exp_samples: [] + - expr: rustfs:storage_snapshot:fresh{observer="n1"} + eval_time: 5m + exp_samples: [] +- name: counters reset independently; filter versions before rate and sum rates across owners + interval: 1m + input_series: + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="n0-disk",api="read_all"} + values: 0 60 120 30 90 150 210 + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 0+60x8 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"} + values: 180x8 + - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",disk_id="n1-disk",api="read_all"} + values: 0+120x6 + - series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",server="n0",drive="/data",observer="old"} + values: 0+999x6 + promql_expr_test: + - expr: sum(resets(rustfs_system_drive_api_calls_total{collection_scope="local"}[5m])) + eval_time: 5m + exp_samples: + - labels: '{}' + value: 1 + - expr: sum(rate(rustfs_system_drive_api_calls_total{collection_scope="local"}[2m]) and ignoring(source_metric) rustfs:storage:current{source_metric="rustfs_system_drive_api_calls_total"}) + eval_time: 5m + exp_samples: + - labels: '{}' + value: 3 +- name: completed slow collection has no validity budget + interval: 1m + input_series: + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"} + values: 0+60x3 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"} + values: '0x3' + promql_expr_test: + - expr: rustfs:storage_snapshot:fresh{observer="slow"} + eval_time: 2m + exp_samples: [] +- name: submillisecond publication and millisecond OTLP samples share a cutoff + interval: 1m + input_series: + - series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"} + values: 0.0009+60x3 + - series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"} + values: 180x3 + - series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector",server="submillisecond",drive="/data"} + values: 100x3 + promql_expr_test: + - expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",observer="submillisecond"}) + eval_time: 2m15s + exp_samples: + - labels: '{}' + value: 1 diff --git a/.github/s3tests/s3tests.conf b/.github/s3tests/s3tests.conf index cc7b51274..945b7b0cd 100644 --- a/.github/s3tests/s3tests.conf +++ b/.github/s3tests/s3tests.conf @@ -55,6 +55,10 @@ secret_key = ${S3_SECRET_KEY} ## replace with key id obtained when secret is created, or delete if KMS not tested #kms_keyid = 01234567-89ab-cdef-0123-456789abcdef +#kms_keyid2 = fedcba98-7654-3210-fedc-ba9876543210 + +## Expected service default for SSE-KMS requests without a key id; empty means none +#kms_default_keyid = ## Storage classes #storage_classes = "LUKEWARM, FROZEN" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0f1034f0..453f1e409 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ name: Continuous Integration on: push: - branches: [ main ] + branches: [ main, release ] paths-ignore: - "**.md" - "docs/**" @@ -36,7 +36,7 @@ on: - "flake.lock" pull_request: types: [ opened, synchronize, reopened, closed ] - branches: [ main ] + branches: [ main, release ] merge_group: types: [ checks_requested ] schedule: @@ -840,13 +840,14 @@ jobs: # Merge gate only (backlog#1149 ci-5): the never-automated user-visible # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption, # security-boundary, ... — via the e2e-full nextest profile. Too heavy for - # every PR, so it is gated to main pushes, the merge queue, and manual + # every PR, so it is gated to main/release pushes, the merge queue, and manual # dispatch. protocols / the 7 cluster suites / replication / #[ignore] are # owned by other lanes (see .config/nextest.toml profile.e2e-full). if: >- github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' || - (github.event_name == 'push' && github.ref == 'refs/heads/main') + (github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release')) needs: [ build-rustfs-debug-binary ] runs-on: sm-standard-2 timeout-minutes: 55 diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index 0e782c228..dea3591a1 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -129,11 +129,22 @@ jobs: : > target/debug/rustfs.features - name: Run upgrade compatibility test + env: + RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }} run: | cargo test --locked -p e2e_test \ "upgrade_compatibility_test::${{ matrix.test }}" \ -- --ignored --exact --nocapture + - name: Upload scanner/heal G09 evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }} + path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }} + if-no-files-found: ignore + retention-days: 14 + - name: Upload server logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/windows-filesystem.yml b/.github/workflows/windows-filesystem.yml index c47e7bc54..918e3f99a 100644 --- a/.github/workflows/windows-filesystem.yml +++ b/.github/workflows/windows-filesystem.yml @@ -16,7 +16,7 @@ name: Windows Filesystem Tests on: push: - branches: [ main ] + branches: [ main, release ] paths: - "crates/ecstore/src/disk/**" - "crates/ecstore/src/store/init_format.rs" @@ -26,7 +26,7 @@ on: - ".github/actions/setup/**" - ".github/workflows/windows-filesystem.yml" pull_request: - branches: [ main ] + branches: [ main, release ] paths: - "crates/ecstore/src/disk/**" - "crates/ecstore/src/store/init_format.rs" diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bac39307..588fb7799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Fresh multi-pool bootstrap with distinct format creators**: a new deployment whose pools have their first endpoint on different nodes (for example two single-node pools) could never publish its initial `pool.bin`: each node held fresh-bootstrap proof only for the pool it formatted, the deployment-wide proof collapsed to none, and every node died with `pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available` after the startup retry budget. The first pool's creator now mints the pending cluster identity on its own pool, every other creator copies that nonce-bound identity onto the pool it formatted first-hand, and the elected writer publishes `pool.bin` once every pool replica carries the same pending identity. Corrupt or disagreeing replicas, pools that merely have a format, expansion pools joining an initialized deployment, and restarts without first-hand proof still fail closed. Non-elected nodes that start before `pool.bin` exists, and the elected writer while it waits for the other creators, no longer latch their pool-metadata write gate for the life of the process. Refs rustfs/backlog#2338, rustfs/backlog#2375. - **Lock RPC timeout storms** (#7363): the remote lock client no longer evicts and re-dials the shared internode HTTP/2 channel on every request deadline. A timeout evicts only when the peer has not completed any lock RPC for two deadlines, evictions and transport-failure re-dials are rate limited per peer (`RUSTFS_OBJECT_LOCK_RPC_EVICTION_COOLDOWN_MS`, default 5 s), and a timed-out request is left running instead of being reset (bounded per peer by `RUSTFS_OBJECT_LOCK_RPC_DETACHED_LIMIT`, default 256), so a slow lock endpoint can no longer drive the `RST_STREAM`/`GOAWAY too_many_resets`/reconnect loop. A lock granted after its caller timed out is released immediately, and unlocks that fail the quick retries continue on a deferred 1/2/4/8/16 s schedule before the server lease reclaims them. New `rustfs_remote_lock_*` metrics cover timeouts, evictions, suppressed evictions, detached streams, late completions and late releases per peer. Operator guide at `docs/operations/lock-rpc-storm-protection.md`. +- **KMS failures on the S3 data path carry an actionable status**: only "key not found" and a backend outage were classified; every other KMS failure — a disabled or pending-deletion key, a denied KMS grant, an encryption-context mismatch, an unsupported algorithm, a credential or timeout failure, a capability the backend does not have — collapsed onto `500 InternalError`. SDKs therefore applied exponential backoff to configuration errors that no retry can fix, and monitoring filed every one of them as a server fault. Unusable-key and request-side failures now return `400`, a denied grant `403`, transient backend failures `503` — including a key store the backend could not read, so an outage stays distinguishable from a missing key all the way to the client — and a missing backend capability `501`. Damaged or unreadable key material still returns `500`, which is what it is. +- **SSE-C on buckets with default encryption**: a `PutObject` carrying a valid SSE-C header triple on a bucket that has default encryption configured no longer fails with `400 InvalidArgument` ("The SSE-C and managed server-side encryption headers cannot be used together"). PUT and the POST-object/extract path resolved the bucket default with a hard-coded "no explicit SSE-C" flag, so the default was layered onto the request and then tripped the request's own mutual-exclusion check; an SSE-C request now suppresses the bucket default on all three write paths, matching COPY and AWS S3. Every bucket with default encryption previously refused SSE-C single PUTs outright, while `CreateMultipartUpload` on the same bucket succeeded. +- **Explicit SSE-S3 on SSE-KMS-default buckets**: `x-amz-server-side-encryption: AES256` against a bucket whose default is `aws:kms` no longer fails with `400 InvalidArgument`. The bucket default's KMS key id was inherited independently of the effective algorithm, producing a self-contradictory `AES256` + key-id pair; the key id is now inherited only when the effective algorithm is `aws:kms`. `PutBucketEncryption` fills in a default key id automatically, so this affected nearly every SSE-KMS-default bucket. +- **Restore of encrypted or compressed multipart objects (silent data corruption)**: restoring a multipart object from a remote tier addressed the tier in *plaintext* coordinates while the copy-back reads the *stored* representation. Every part received a misaligned slice of the remote object whose length still satisfied the range, the hash reader and the completion size check, so the restore reported success and replaced the object's bytes. Restore now accumulates stored part sizes, passes the stored length to the hash reader alongside the plaintext length, and validates against the stored size. Objects restored by an affected release must be re-restored from the tier or recovered from a backup — this release does not detect or repair them retroactively. +- **Restore no longer drifts the object ETag**: the copy-back digests stored (encrypted or compressed) bytes, so the recomputed MD5 is not the object's public ETag. Single-part and multipart restores now preserve the original object ETag, and each restored part keeps its own recorded part ETag. +- **ILM archive no longer forwards encryption metadata to the tier**: transition requests carried the object's SSE headers and the RustFS-wrapped data key as request headers. Any S3 target rejected an SSE-C archive outright (`400`, no key supplied), an SSE-KMS archive asked the target to encrypt a second time under a key id it does not own, and the wrapped DEK left the cluster. The archive request now strips every SSE header and encryption marker using the same predicate the replication path uses; the local `xl.meta` keeps all of it, so read-through and restore are unaffected. +- **KMS reload is no longer a no-op on a node whose KMS failed to start**: `POST /rustfs/admin/v3/kms/reload` short-circuited whenever the persisted configuration matched the in-memory one byte for byte. A node whose KMS failed to start (for example Vault briefly unreachable during a rolling restart) keeps that configuration and sits in `Error`, so the documented recovery call returned "reloaded successfully" while leaving the node down — and did the same on every peer through the reload broadcast. Reload now short-circuits only for a service that is actually running, and otherwise reconfigures, which starts the service. +- **AWS KMS capability reporting**: the AWS backend no longer advertises `versioning` support through `GET /rustfs/admin/v3/kms/status`. AWS KMS key versions are not enumerable through this backend, as the backend documentation already stated. - **Multipart admission queue**: an `UploadPart` waiting for a foreground write permit now waits at most 10 s by default (`RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_WAIT_TIMEOUT_MS`, previously 30 s), so a queued part returns S3 `SlowDown` before the client's socket write timeout drops the connection. Separately, the API listener no longer forces a 4 MiB `SO_RCVBUF` on every accepted socket (kernel autotuning applies; `RUSTFS_HTTP_SOCKET_RECV_BUFFER_BYTES` restores a fixed size), so a queued part no longer lets up to 8 MiB of unread body accumulate in kernel memory per connection, which is what throttled whole nodes under SDK-default multipart concurrency. Fixes #7385. - **Helm Ingress**: `customAnnotations` are now merged with class-specific annotations (nginx/traefik) instead of being ignored when `ingress.className` is set. - **Per-pool erasure parity**: Erasure parity (STANDARD and reduced-redundancy) is now resolved independently for every pool instead of reusing the first pool's value. A heterogeneous topology — for example a 4-drive pool plus a 2-drive pool created during expansion — previously inherited the first pool's parity and could resolve to zero data shards in the smaller pool, panicking Reed-Solomon construction on write. Automatic parity now resolves per pool (for example `2+2` in the 4-drive pool and `1+1` in the 2-drive pool). Fixes #4801. @@ -64,6 +72,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Four-layer regression-prevention tests guard against silent feature deletion: compile-time module assertion, module-presence unit test, cross-module `Protocol` enum assertion, end-to-end SSH banner test against the running binary ### Changed +- **Encryption and KMS work merged since `1.0.0-rc.5`** (entries were missing from this section): + - **Persisted KMS configuration secrets** are sealed field-by-field with `RUSTFS_KMS_CONFIG_SECRET`. **When the variable is unset the secrets are persisted in cleartext and the server only warns** (`persisted KMS configuration carries cleartext secrets`); it never refuses the write. Set it, identically, on every node, and re-save the configuration to seal an existing one. + - **New v2 ciphertext frame format** with per-frame index binding and final-frame authentication. Its write switch `RUSTFS_ENCRYPTION_FRAME_V2` is **off by default**: v2 frames are unreadable by nodes without v2 read support, and encrypted ciphertext travels verbatim through transition, decommission and SSE-C replication passthrough, so turn it on only after every node — and every RustFS warm/replication target that receives raw ciphertext — runs a release with v2 read support. Reading v2 objects needs no switch. + - **Per-key SSE-KMS authorization** (`RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY`, default `false`). With it on, anonymous callers hold no KMS grants, so **a public bucket serving SSE-KMS objects is an incompatible combination** and those reads return `AccessDenied`. + - Envelope context binding as KMS AAD (`ENV_KMS_ENVELOPE_AAD`, off by default; a node that predates the field cannot open bound envelopes). + - Vault custom CA and mutual TLS; object-level DEK rewrap plus a batch rekey admin API; a backend-locality runtime signal on `kms/status`. + - Single-pass decryption for encrypted GET, and encrypted single-part closed-range seek — the latter is now **on by default** (`RUSTFS_ENCRYPTED_RANGE_SEEK`, default `true`; the switch remains as a kill switch). +- **Vault static tokens are now tracked and renewed**: with `Token` authentication RustFS hard-coded "this token has no lease", so the renewal task never started and no remaining-TTL gauge was published. `vault token create` grants a 768-hour TTL by default, which turned a healthy-looking cluster into one where every KMS call returned 403 about a month later, with no self-healing short of a restart or reconfigure. RustFS now calls `auth/token/lookup-self` at login and adopts what Vault reports: a non-expiring token behaves exactly as before, an expiring renewable one is renewed at half TTL like the other auth methods, and an expiring non-renewable one logs `vault_static_token_not_renewable` and publishes its remaining TTL. The probe never fails the login: a token whose policy omits `lookup-self` (Vault's `default` policy grants it), or a Vault that is unreachable at that moment, logs `vault_static_token_lookup_failed` and falls back to the previous no-lease behaviour, so no deployment that works today stops working. +- **SSE-C over a plaintext transport is reported**: AWS S3 and MinIO refuse an SSE-C request that did not arrive over TLS, because the customer key travels in a request header. RustFS accepted them on any transport and still does by default — flipping to a rejection inside a release window would break plaintext staging and test deployments. Each such request now increments `rustfs_ssec_plaintext_requests_total` and logs one `ssec_request_without_tls` warning per process, and `RUSTFS_SSE_C_REQUIRE_TLS=true` opts into the AWS `400` now. The default is expected to flip in a later release; confirm the counter reads zero first. The verdict is per connection: a TLS listener satisfies it, and so does an `https` protocol forwarded by a proxy the trusted-proxy configuration accepts. +- **Local KMS backend on a distributed deployment says what actually breaks**: the backend keeps key material and its Argon2id salt on each node's own disk, so two nodes derive different keys from the same `master_key` and an object encrypted on one node cannot be decrypted on another — intermittent 500s behind a load balancer. Configuring it while the deployment is distributed now logs `kms_node_local_backend_in_distributed_deployment` and appends that consequence to the `kms/configure` response, instead of only the generic "development only" positioning warning. It remains a warning, not a gate. +- **SSE-KMS is refused when no KMS is running (breaking)**: a write requesting `x-amz-server-side-encryption: aws:kms` on a node with no KMS service no longer succeeds. Earlier releases wrapped the data key with the node-local `RUSTFS_SSE_S3_MASTER_KEY` while still writing `aws:kms` and the requested key id into the object metadata — metadata that claimed a KMS protection the object never had, under a key that was never consulted. Such a request now returns `400 InvalidRequest` when KMS was never configured and `503` when a configured service is not running; the refusal is evaluated after the per-key authorization gate, so an unauthorized caller still receives `403 AccessDenied`. **Upgrade note:** a deployment that relied on this write succeeding will start receiving 4xx/503. Either configure a KMS, or request `AES256` and keep the documented SSE-S3 local-master-key fallback, which is unchanged. Objects already written this way remain readable. +- **Legacy ciphertext nonce layouts are now locked per segment**: while decrypting a v1 segment, the reader locks onto whichever of the three historical nonce layouts decoded the segment's first non-zero-index frame and rejects any later frame that needs a different one. Because a frame encrypted at block index zero authenticates under the pre-`1.0.0-alpha.91` reused-part-nonce layout at any position, an attacker able to rewrite the underlying shards could previously replay it and have the forged plaintext returned with `200`. New `RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK` (default `true`) drops that third layout entirely when set to `false`, which closes the residual case of a stream built purely from repeats of frame zero. Turn it off only after migrating pre-alpha.91 encrypted objects (rewrite in place with CopyObject); see [KMS backend security properties](docs/operations/kms-backend-security.md) for what the v1 frame layout does and does not authenticate. - **HTTP Server Stack**: Integrated `KeystoneAuthLayer` middleware from `rustfs-keystone` crate into service stack (positioned after ReadinessGateLayer) - **Storage-class validation on startup (upgrade note)**: A persisted explicit storage class (`RUSTFS_STORAGE_CLASS_STANDARD` / `RUSTFS_STORAGE_CLASS_RRS`, for example `EC:2`) is now validated against the actual per-pool drive counts at startup and rejected when a pool cannot satisfy it. This is fail-closed and correct, but a cluster that persisted a storage class larger than a small or heterogeneous pool can hold (for example `EC:2` alongside a 2-drive pool), which earlier releases accepted and silently resolved to an invalid layout, will now refuse to start after upgrade. To recover, unset `RUSTFS_STORAGE_CLASS_STANDARD` so the server derives a valid per-pool default automatically, or set it to a value every pool can satisfy. - **IAMAuth**: Enhanced `get_secret_key()` to return empty secret for Keystone credentials (bypasses signature validation) diff --git a/Cargo.lock b/Cargo.lock index d921d6b66..c00244594 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -282,7 +282,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -701,9 +701,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.44" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844" +checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" dependencies = [ "compression-codecs", "compression-core", @@ -1031,9 +1031,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.145.0" +version = "1.146.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93" +checksum = "37b5ffaae346b9bd486ebdc3eb7053ec8ab8a1ad0f19a332eefeff036ed559e6" dependencies = [ "arc-swap", "aws-credential-types", @@ -1641,9 +1641,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] @@ -1735,18 +1735,18 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" dependencies = [ "darling 0.24.1", "ident_case", @@ -1906,7 +1906,7 @@ dependencies = [ "maybe-owned", "rustix", "rustix-linux-procfs", - "windows-sys 0.61.2", + "windows-sys 0.60.2", "winx", ] @@ -2220,7 +2220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "unicode-segmentation", - "unicode-width 0.2.2", + "unicode-width", ] [[package]] @@ -2238,9 +2238,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.39" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" +checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" dependencies = [ "brotli 8.0.4", "bzip2", @@ -2249,8 +2249,8 @@ dependencies = [ "liblzma", "lz4", "memchr", - "zstd 0.13.3", - "zstd-safe 7.3.0", + "zstd 0.14.0", + "zstd-safe 8.0.0", ] [[package]] @@ -2270,9 +2270,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.4" +version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" dependencies = [ "encode_unicode", "libc", @@ -2408,6 +2408,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpp_demangle" version = "0.5.1" @@ -3927,16 +3933,6 @@ dependencies = [ "dirs-sys", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - [[package]] name = "dirs-sys" version = "0.5.0" @@ -3945,19 +3941,8 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.2", - "windows-sys 0.61.2", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users 0.4.6", - "winapi", + "redox_users", + "windows-sys 0.59.0", ] [[package]] @@ -3966,7 +3951,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "objc2", ] @@ -4208,11 +4193,17 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" dependencies = [ "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -4296,7 +4287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4425,7 +4416,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "rustc_version", ] @@ -5071,9 +5062,9 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" dependencies = [ "hashbrown 0.17.1", ] @@ -5287,9 +5278,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592" +checksum = "366c5db5a8e4643bdde121a1d6f35cfa035e7850b8b8ea94c6960a139d44e93a" dependencies = [ "arc-swap", "async-channel", @@ -5298,6 +5289,7 @@ dependencies = [ "crossbeam-channel", "flate2", "futures-channel", + "futures-core", "futures-util", "hdrhistogram", "hotpath-macros", @@ -5306,9 +5298,8 @@ dependencies = [ "object 0.36.7", "parking_lot", "pin-project-lite", - "prettytable-rs", "quanta", - "regex", + "regex-lite", "reqwest", "reqwest-middleware", "rustc-demangle", @@ -5410,9 +5401,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "ctutils", "subtle", @@ -5711,11 +5702,11 @@ checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" [[package]] name = "io-uring" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "libc", ] @@ -5751,17 +5742,6 @@ dependencies = [ "serde", ] -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_debug" version = "1.1.0" @@ -6021,9 +6001,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" [[package]] name = "lapin" -version = "4.10.0" +version = "4.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd20e01fd92597ca352ca7ceed3c589851ebad279dfcada48aa4d24fd3a7caa" +checksum = "24b99d8cdfe3f6223f9f75e35e7510a8662767bee27c1cf870d503fc3bb2428a" dependencies = [ "amq-protocol", "async-rs", @@ -6232,7 +6212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9f8ff371890db2cf65a0758dba9a79f9cd965de369f6dbdc6581a22780af45e" dependencies = [ "async-trait", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "dashmap", @@ -6724,6 +6704,33 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "multiversion" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn 3.0.5", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "murmur3" version = "0.5.2" @@ -6787,7 +6794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.1", + "bitflags 2.13.2", "btoi", "byteorder", "bytes", @@ -6819,7 +6826,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "byteorder", "derive_builder", "getset", @@ -6867,7 +6874,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -6880,7 +6887,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -6892,7 +6899,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -6956,7 +6963,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7093,7 +7100,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d164abbde0b3c03edb9edb9cb8d31a7f5b79015c692b7c771f6e0840e9106b9f" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "libloading", "nvml-wrapper-sys", "static_assertions", @@ -7144,7 +7151,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "dispatch2", "objc2", ] @@ -7161,7 +7168,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "objc2", ] @@ -7709,12 +7716,12 @@ dependencies = [ [[package]] name = "pe-unwind-info" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97f6fccfd2d9d2df765ca23ff85fe5cc437fb0e6d3e164e4d3cbe09d14780c93" +checksum = "e33c6dbf1a8fb7f71742cd70f5e9f0986e60b2d19dc0b28d9ca0d1323259274a" dependencies = [ "arrayvec", - "bitflags 2.13.1", + "bitflags 2.13.2", "thiserror 2.0.20", "zerocopy", "zerocopy-derive", @@ -7770,7 +7777,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "575828d9d7d205188048eb1508560607a03d21eafdbba47b8cade1736c1c28e1" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "c-enum", "perf-event-open-sys2", ] @@ -8187,19 +8194,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "prettytable-rs" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" -dependencies = [ - "encode_unicode", - "is-terminal", - "lazy_static", - "term", - "unicode-width 0.1.14", -] - [[package]] name = "primefield" version = "0.14.0" @@ -8300,7 +8294,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec 0.8.0", - "bitflags 2.13.1", + "bitflags 2.13.2", "num-traits", "rand 0.9.5", "rand_chacha 0.9.0", @@ -8442,7 +8436,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "memchr", "unicase", ] @@ -8519,24 +8513,24 @@ dependencies = [ [[package]] name = "qrcode-core" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79de8ed9b3c65c866e0ca79ba3aa8b05264e5183b0f894f1ea43b3a53146d657" +checksum = "5a5c0d85fe467f79319e9f0d0ffe43f8c32defae67263f78d57e33d8b4a638bf" [[package]] name = "qrcode-decode" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c307d82a6b418dc99004a68cdd621c0606874faae8645036eef9168ec5aef4" +checksum = "04cf729caefa8fd0919f8d111a4e6531942ac48742e7649ed4c9c159b41a6fda" dependencies = [ "qrcode-core", ] [[package]] name = "qrcode-eps" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cf7d12b9e539644309c628db9436be7024e0fbe16f440109ce92d7c9787688" +checksum = "d289a6212464685d1e598c3692e02189a096b2f4abe1acdb1f4fba6a932809d2" dependencies = [ "qrcode-core", "qrcode-render", @@ -8544,9 +8538,19 @@ dependencies = [ [[package]] name = "qrcode-html" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb7d790f2b34ce1edbafdf7a6802e73694aa94798be275c63acb780a39c1e6c" +checksum = "5a14b98d4aa6278f7e7218ad901837a85cef040a262127d0c4b6f8d3bd51d46e" +dependencies = [ + "qrcode-core", + "qrcode-render", +] + +[[package]] +name = "qrcode-image" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7923bee1dc13ef96b9778e9299d803ae9f5de38971cb8e55c141022cc296618" dependencies = [ "qrcode-core", "qrcode-render", @@ -8554,15 +8558,15 @@ dependencies = [ [[package]] name = "qrcode-parse" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f6484a2329e87b5a4b21c983d68bee32a442087287c0679efe782689ee469a" +checksum = "0726e7e2acd6335b63936ba1d784361765313f53274942efb0636407e882fdc9" [[package]] name = "qrcode-pdf" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deda8c86193091e31856024288e986869559ace2a71b7b477f59b4caea4dabf4" +checksum = "41ed9a51cab153e441963f8ec64ffc980d7a40bfd7fa8f6e0edd290424cda206" dependencies = [ "qrcode-core", "qrcode-render", @@ -8570,9 +8574,9 @@ dependencies = [ [[package]] name = "qrcode-pic" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bda277eb6e11b06e0ebe0cb1b519d71467a3f1d7f6408336af9a979edb1091" +checksum = "038be675ee01aeae82f20eea4b043054f8232e14d3ccc7836803510ad3ff6524" dependencies = [ "qrcode-core", "qrcode-render", @@ -8580,23 +8584,24 @@ dependencies = [ [[package]] name = "qrcode-render" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a40771febec53bdf7de3b3de6549950a7c9f54eef58477d99f4de2fd9f114f9" +checksum = "7af1ddfd0f2496c28276913e26289e8fb245cab16ece7efcf9df46e2027dd3c8" dependencies = [ "qrcode-core", ] [[package]] name = "qrcode-rs" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62237103e62d01195453a4b34d56a35f59cf8c8ca3fa8ddcfca84336d5c4a2a" +checksum = "246e31e4a82d59d4714c589e4912f27bc67f5fa205cc4e3bdf916ccc640af010" dependencies = [ "qrcode-core", "qrcode-decode", "qrcode-eps", "qrcode-html", + "qrcode-image", "qrcode-parse", "qrcode-pdf", "qrcode-pic", @@ -8606,9 +8611,9 @@ dependencies = [ [[package]] name = "qrcode-svg" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513f1dd2a0e11d1bcc4dd3268887cdad059ed8ab5269d45ce57aada1256c5405" +checksum = "19f641cac9a021d31948437c740baec47243c03586c3230c839b76df9e4df084" dependencies = [ "qrcode-core", "qrcode-render", @@ -8707,7 +8712,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8869,7 +8874,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", ] [[package]] @@ -8978,18 +8983,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.1", -] - -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 1.0.69", + "bitflags 2.13.2", ] [[package]] @@ -9084,11 +9078,11 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "encoding_rs", "futures-channel", @@ -9310,13 +9304,13 @@ dependencies = [ [[package]] name = "russh" -version = "0.63.2" +version = "0.63.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd" +checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce" dependencies = [ "aes 0.9.3", "aws-lc-rs", - "bitflags 2.13.1", + "bitflags 2.13.2", "block-padding 0.4.2", "byteorder", "bytes", @@ -9393,11 +9387,11 @@ dependencies = [ [[package]] name = "russh-sftp" -version = "2.4.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de67aace74530a29086db0671fa200c470a58eb380081f28ad512ffb0c5356b" +checksum = "093197e526668d92bba562e2bbbe98d1af9831bf080b619c736316ca1fa35101" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "chrono", "dashmap", @@ -9928,6 +9922,7 @@ dependencies = [ "async-trait", "base64-simd", "bytes", + "chrono", "crc-fast", "futures", "hotpath", @@ -10578,6 +10573,7 @@ dependencies = [ "serde_json", "sha1 0.11.0", "sha2 0.11.0", + "temp-env", "thiserror 2.0.20", "tokio", "tokio-test", @@ -11065,11 +11061,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11084,9 +11080,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" dependencies = [ "aws-lc-rs", "log", @@ -11152,7 +11148,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11435,7 +11431,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -11602,11 +11598,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", @@ -11623,14 +11619,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -11953,7 +11949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12104,7 +12100,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12194,9 +12190,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "suppaftp" -version = "11.0.0" +version = "12.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a" +checksum = "9c6978f1b04fc86f122f767b88a9d1d89e80cfe3dc97d670598507d713c4ce48" dependencies = [ "async-trait", "chrono", @@ -12325,7 +12321,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -12409,21 +12405,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -12891,7 +12876,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-util", "http 1.5.0", @@ -12910,7 +12895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ "async-compression", - "bitflags 2.13.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -13192,12 +13177,6 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.2" @@ -13265,9 +13244,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13531,7 +13510,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -13841,7 +13820,7 @@ version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "windows-sys 0.59.0", ] @@ -13998,18 +13977,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c12edcaba..4680a378e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,7 +145,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" } async-channel = "2.5.0" async_zip = { default-features = false, version = "0.0.19" } mysql_async = { default-features = false, version = "0.37.1" } -async-compression = { version = "0.4.44" } +async-compression = { version = "0.4.46" } async-recursion = "1.1.1" async-trait = "0.1.92" async-nats = { version = "0.50.0", default-features = false } @@ -156,7 +156,7 @@ futures-lite = "2.6.1" futures-util = "0.3.34" pollster = "1.0.1" pulsar = { default-features = false, version = "6.9.0" } -lapin = { default-features = false, version = "4.10.0" } +lapin = { default-features = false, version = "4.11.0" } hyper = { version = "1.11.1" } hyper-rustls = { default-features = false, version = "0.27.9" } hyper-util = { version = "0.1.20" } @@ -164,7 +164,7 @@ http = "1.5.0" http-body = "1.1.0" http-body-util = "0.1.5" minlz = "1.2.3" -reqwest = "0.13.4" +reqwest = "0.13.5" rustfs-kafka-async = { version = "1.3.1" } socket2 = { version = "0.6.5" } tokio = { version = "1.53.1" } @@ -211,7 +211,7 @@ openidconnect = { default-features = false, version = "4.0" } pbkdf2 = "0.13.0" p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] } rsa = { version = "=0.10.0-rc.18" } -rustls = { default-features = false, version = "0.23.43" } +rustls = { default-features = false, version = "0.23.44" } rustls-native-certs = "0.8" rustls-pki-types = "1.15.1" x509-parser = "0.18.1" @@ -245,7 +245,7 @@ atomic_enum = "0.3.0" aws-config = { version = "1.12.0" } aws-credential-types = { version = "1.3.0" } aws-sdk-kms = { default-features = false, version = "1.118.0" } -aws-sdk-s3 = { default-features = false, version = "1.145.0" } +aws-sdk-s3 = { default-features = false, version = "1.146.0" } aws-sdk-sts = { default-features = false, version = "1.114.0" } aws-smithy-async = { version = "1.3.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } @@ -297,7 +297,7 @@ percent-encoding = "2.3.2" # Server-side QR rendering for TOTP enrollment, so neither the console nor the # CLI needs its own QR encoder. No default features: the image/render backends # pull in an image stack this only needs SVG and text output from. -qrcode-rs = { version = "2.0.0", default-features = false, features = ["std", "svg"] } +qrcode-rs = { version = "2.1.0", default-features = false, features = ["std", "svg"] } pin-project-lite = "0.2.17" pretty_assertions = "1.4.1" rand = { version = "0.10.2" } @@ -335,7 +335,7 @@ tracing-subscriber = { version = "0.3.23" } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.26.0" } +uuid = { version = "1.26.1" } vaultrs = { version = "0.8.0" } tar = "0.4.46" walkdir = "2.5.0" @@ -362,10 +362,10 @@ pyroscope = { version = "2.1.1" } # FTP and SFTP libunftp = { version = "0.23.0" } unftp-core = "0.1.0" -suppaftp = { version = "11.0.0" } +suppaftp = { version = "12.0.0" } rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.63.2" } -russh-sftp = "2.4.0" +russh = { version = "0.63.3" } +russh-sftp = "3.0.0" # WebDAV dav-server = "0.11.0" @@ -373,7 +373,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling rustfs-mimalloc = { version = "0.5.3" } # Preserve Unicode focus filters until rustfs/backlog#2302 is resolved. -hotpath = { version = "=0.25.0", default-features = false } +hotpath = { version = "0.25.1", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/README_ZH.md b/README_ZH.md index 2951a5ddf..c0f9dde19 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -46,14 +46,31 @@ RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。Rust - **完全开源**：采用 Apache 2.0 许可证，鼓励社区贡献和商业使用。 - **简单易用**：设计简洁，易于部署和管理。 -| 功能 | 状态 | 功能 | 状态 | -| :----------------- | :------ | :---------------------- | :-------- | -| **S3 核心功能** | ✅ 可用 | **Bitrot (防数据腐烂)** | ✅ 可用 | -| **上传 / 下载** | ✅ 可用 | **单机模式** | ✅ 可用 | -| **版本控制** | ✅ 可用 | **存储桶复制** | ✅ 可用 | -| **日志功能** | ✅ 可用 | **生命周期管理** | 🚧 测试中 | -| **事件通知** | ✅ 可用 | **分布式模式** | 🚧 测试中 | -| **K8s Helm Chart** | ✅ 可用 | **OPA (策略引擎)** | 🚧 测试中 | +状态说明：✅ 可用 —— 已发布并有 CI 门禁覆盖；🧪 预览 —— 已发布但需显式开关，或兼容性承诺有边界。 + +| 功能 | 状态 | 功能 | 状态 | +| :-------------------------- | :------ | :----------------------- | :------ | +| **S3 核心功能** | ✅ 可用 | **分布式模式** | ✅ 可用 | +| **上传 / 下载** | ✅ 可用 | **单机模式** | ✅ 可用 | +| **版本控制** | ✅ 可用 | **Bitrot (防数据腐烂)** | ✅ 可用 | +| **对象锁定 (WORM)** | ✅ 可用 | **修复与扫描器** | ✅ 可用 | +| **服务端加密 (SSE)** | ✅ 可用 | **存储池扩容 / 下线** | ✅ 可用 | +| **RustFS KMS** | ✅ 可用 | **存储桶复制** | ✅ 可用 | +| **生命周期管理 (ILM)** | ✅ 可用 | **站点复制** | ✅ 可用 | +| **ILM 分层 (远端 S3)** | ✅ 可用 | **存储桶配额** | ✅ 可用 | +| **S3 Select** | ✅ 可用 | **事件通知** | ✅ 可用 | +| **S3 Tables (Iceberg REST)**| 🧪 预览 | **审计日志** | ✅ 可用 | +| **IAM / 策略** | ✅ 可用 | **日志与可观测性** | ✅ 可用 | +| **OIDC / SSO** | ✅ 可用 | **Web 控制台** | ✅ 可用 | +| **Keystone 认证** | ✅ 可用 | **K8s Helm Chart** | ✅ 可用 | +| **Swift API** | ✅ 可用 | **FTPS / WebDAV** | ✅ 可用 | +| **多租户** | ✅ 可用 | **SFTP** | ✅ 可用 | +| **MinIO 磁盘格式兼容** | 🧪 预览 | | | + +说明： + +- **服务端加密**：支持 SSE-C、SSE-S3 与 SSE-KMS。SSE-KMS 必须先配置 KMS 服务；未配置 KMS 时请求 `aws:kms` 会被拒绝，不会降级到本地主密钥。 +- **RustFS KMS**：生产环境支持 Vault（KV2 / Transit）与 AWS KMS 后端；`Local` 与 `Static` 后端仅供开发与测试使用，详见 [KMS 后端安全属性](docs/operations/kms-backend-security.md)。 ## RustFS vs MinIO 性能对比 diff --git a/crates/common/src/mrf_channel.rs b/crates/common/src/mrf_channel.rs index 7f782bebf..c6bf28da3 100644 --- a/crates/common/src/mrf_channel.rs +++ b/crates/common/src/mrf_channel.rs @@ -145,6 +145,38 @@ pub fn consume_verified_mrf_repair_events(anchors: &mut Vec) -> usize { + let Some(registry) = MRF_VERIFIED_REPAIR_EVENTS.get() else { + return 0; + }; + let Ok(mut events) = registry.lock() else { + return 0; + }; + let before = anchors.len(); + let mut retained = std::collections::VecDeque::with_capacity(events.len()); + while let Some(event) = events.pop_front() { + if event.bucket.as_ref() != bucket { + retained.push_back(event); + continue; + } + let mut matched = false; + anchors.retain(|anchor| { + let proven = anchor.is_proven_by(&event); + matched |= proven; + !proven + }); + if !matched { + retained.push_back(event); + } + } + *events = retained; + before.saturating_sub(anchors.len()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct MrfScope { pub pool_index: u32, @@ -821,6 +853,71 @@ mod tests { assert!(retained.is_empty()); } + #[test] + fn recorded_verified_repair_consumer_retains_unmatched_proofs() { + let bucket = Arc::::from(format!("recorded-proof-{}", Uuid::new_v4())); + let other_bucket = Arc::::from(format!("recorded-proof-other-{}", Uuid::new_v4())); + let incarnation = Uuid::new_v4(); + let lease = MrfIngressLease::new(21); + let retained_anchor = MrfDurableRepairAnchor { + kind: MrfKind::PartialWrite, + bucket: bucket.clone(), + object: Arc::from("retained"), + version_id: Some([7; 16]), + scope: Some(MrfScope { + pool_index: 1, + set_index: 2, + }), + lease, + bucket_incarnation_id: incarnation, + }; + let waiting_anchor = MrfDurableRepairAnchor { + object: Arc::from("waiting"), + lease: MrfIngressLease::new(22), + ..retained_anchor.clone() + }; + let matched_event = MrfVerifiedRepairEvent { + kind: retained_anchor.kind, + bucket: bucket.clone(), + object: retained_anchor.object.clone(), + version_id: retained_anchor.version_id, + scope: retained_anchor.scope, + lease: Some(retained_anchor.lease), + bucket_incarnation_id: retained_anchor.bucket_incarnation_id, + disposition: MrfVerifiedRepairDisposition::Repaired, + }; + let same_bucket_unmatched = MrfVerifiedRepairEvent { + object: Arc::from("future"), + lease: Some(MrfIngressLease::new(23)), + ..matched_event.clone() + }; + let other_bucket_event = MrfVerifiedRepairEvent { + bucket: other_bucket.clone(), + ..matched_event.clone() + }; + + note_mrf_verified_repair(matched_event); + note_mrf_verified_repair(same_bucket_unmatched.clone()); + note_mrf_verified_repair(other_bucket_event.clone()); + + let mut anchors = vec![retained_anchor, waiting_anchor.clone()]; + assert_eq!(consume_recorded_verified_mrf_repair_events_for(&bucket, &mut anchors), 1); + assert_eq!(anchors, vec![waiting_anchor]); + + let remaining_bucket_events = take_mrf_verified_repair_events_for(&bucket); + assert_eq!( + remaining_bucket_events, + vec![same_bucket_unmatched], + "same-bucket proofs without a retained anchor must remain available" + ); + let remaining_other_events = take_mrf_verified_repair_events_for(&other_bucket); + assert_eq!( + remaining_other_events, + vec![other_bucket_event], + "proofs for other buckets must not be drained by this consumer" + ); + } + #[tokio::test] async fn try_send_delivers_and_respects_capacity() { let mut receiver = init_mrf_channel().expect("first initialization should succeed"); diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index 68a4d9822..3b0e81699 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -593,6 +593,20 @@ pub struct DataUsageSnapshotIdentity { pub scanner_epoch: Option, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct DataUsageSegmentInvalidationProof { + #[serde(default)] + pub process_epoch: String, + #[serde(default)] + pub generation_start: u64, + #[serde(default)] + pub generation_end: u64, + #[serde(default)] + pub producer_identity_coverage_complete: bool, + #[serde(default)] + pub cold_zero_walk_oracle: bool, +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct DataUsageSnapshotSetState { pub pool_index: u64, @@ -607,6 +621,8 @@ pub struct DataUsageSnapshotSetState { pub complete: bool, #[serde(default)] pub tombstone: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub segment_invalidation_proof: Option, } impl DataUsageInfo { @@ -3073,6 +3089,7 @@ mod tests { scan_plan_digest: Some([1; 32]), complete: false, tombstone: false, + segment_invalidation_proof: None, }]; assert!(observed_data_usage_is_newer(&partial, &authoritative)); } @@ -3095,6 +3112,7 @@ mod tests { scan_plan_digest: Some([1; 32]), complete: true, tombstone: false, + segment_invalidation_proof: None, }, DataUsageSnapshotSetState { pool_index: 1, @@ -3104,6 +3122,7 @@ mod tests { scan_plan_digest: Some([2; 32]), complete: false, tombstone: false, + segment_invalidation_proof: None, }, ], ..Default::default() @@ -3113,6 +3132,42 @@ mod tests { assert!(partial.is_valid_partial_snapshot()); } + #[test] + fn set_state_segment_invalidation_proof_is_additive() { + #[derive(Deserialize)] + struct LegacySetState { + pool_index: u64, + set_index: u64, + complete: bool, + } + + let proof = DataUsageSegmentInvalidationProof { + process_epoch: "scanner-process".to_string(), + generation_start: 3, + generation_end: 5, + producer_identity_coverage_complete: true, + cold_zero_walk_oracle: true, + }; + let state = DataUsageSnapshotSetState { + pool_index: 1, + set_index: 2, + scanner_cycle: Some(9), + scanner_epoch: Some(4), + scan_plan_digest: Some([7; 32]), + complete: true, + tombstone: false, + segment_invalidation_proof: Some(proof.clone()), + }; + let encoded = rmp_serde::to_vec_named(&state).expect("set state should encode with additive proof"); + let legacy: LegacySetState = rmp_serde::from_slice(&encoded).expect("legacy readers should ignore proof metadata"); + assert_eq!(legacy.pool_index, 1); + assert_eq!(legacy.set_index, 2); + assert!(legacy.complete); + + let decoded: DataUsageSnapshotSetState = rmp_serde::from_slice(&encoded).expect("new readers should restore proof"); + assert_eq!(decoded.segment_invalidation_proof, Some(proof)); + } + #[test] fn completeness_marker_requires_a_snapshot_timestamp() { let untimestamped = DataUsageInfo { diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index 9ba6ae346..28da958bc 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -30,6 +30,8 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern: | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup | | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory | +The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command. + ## How to run All commands assume repo root. `cargo test` triggers an on-demand build of the diff --git a/crates/e2e_test/build.rs b/crates/e2e_test/build.rs index 6d412d3d4..70437aea7 100644 --- a/crates/e2e_test/build.rs +++ b/crates/e2e_test/build.rs @@ -51,7 +51,7 @@ fn main() { } } let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned()); - let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty()); + let dirty = git(&root, &["status", "--porcelain", "--untracked-files=no"]).is_none_or(|status| !status.is_empty()); let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned()); let mut features = std::env::vars() .filter_map(|(key, _)| { diff --git a/crates/e2e_test/src/cluster_concurrency_test.rs b/crates/e2e_test/src/cluster_concurrency_test.rs index 03d73954a..177cdbd8f 100644 --- a/crates/e2e_test/src/cluster_concurrency_test.rs +++ b/crates/e2e_test/src/cluster_concurrency_test.rs @@ -268,6 +268,7 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<( let rule = CorsRule::builder() .allowed_methods("GET") .allowed_origins("https://example.com") + .allowed_headers("*") .build()?; let configuration = CorsConfiguration::builder().cors_rules(rule).build()?; @@ -288,6 +289,60 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<( assert_eq!(rules[0].allowed_methods(), ["GET"]); assert_eq!(rules[0].allowed_origins(), ["https://example.com"]); + let http = reqwest::Client::builder().no_proxy().build()?; + let url = format!("http://{}/{}", cluster.nodes[1].address, BUCKET_METADATA_RELOAD_BUCKET); + let without_headers = http + .request(reqwest::Method::OPTIONS, &url) + .header("Origin", "https://example.com") + .header("Access-Control-Request-Method", "GET") + .send() + .await?; + assert!(without_headers.status().is_success()); + assert!(!without_headers.headers().contains_key("access-control-allow-headers")); + assert!( + without_headers + .headers() + .get("vary") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("Access-Control-Request-Headers")), + "a cached header-free preflight must not suppress a later requested header grant" + ); + let preflight = http + .request(reqwest::Method::OPTIONS, &url) + .header("Origin", "https://example.com") + .header("Access-Control-Request-Method", "GET") + .header("Access-Control-Request-Headers", "X-Another-Header, x-could-be-anything") + .send() + .await?; + assert!(preflight.status().is_success(), "peer preflight should succeed: {preflight:?}"); + assert_eq!( + preflight + .headers() + .get("access-control-allow-headers") + .and_then(|value| value.to_str().ok()), + Some("x-another-header,x-could-be-anything"), + "a wildcard rule must return only the headers requested by this preflight" + ); + assert!( + preflight + .headers() + .get("vary") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("Access-Control-Request-Headers")), + "preflight caches must distinguish the requested header list" + ); + let denied = http + .request(reqwest::Method::OPTIONS, &url) + .header("Origin", "https://disallowed.example.com") + .header("Access-Control-Request-Method", "GET") + .header("Access-Control-Request-Headers", "x-another-header") + .send() + .await?; + assert!( + !denied.headers().contains_key("access-control-allow-headers"), + "a rejected origin must not receive the requested header grant" + ); + writer .delete_bucket_cors() .bucket(BUCKET_METADATA_RELOAD_BUCKET) diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index ed4ba96b6..2c3aa9631 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -188,6 +188,15 @@ fn write_next_test_port(port: u16) -> Result<(), Box) -> String { + let max_inclusive = config.max_exclusive() - 1; + let detail = last_error.map(|err| format!("; last bind error: {err}")).unwrap_or_default(); + format!( + "no available E2E test port found in {}..={} after {} attempts{}", + config.min, max_inclusive, attempts, detail + ) +} + pub(crate) fn capture_command_logs( command: &mut Command, log_path: Option<&str>, @@ -694,19 +703,23 @@ impl RustFSTestEnvironment { let _guard = PortAllocatorGuard::acquire().await?; let config = test_port_allocator_config()?; let mut next_port = read_next_test_port(config); + let mut last_error = None; for _ in 0..config.range { let port = next_port; next_port = advance_test_port(next_port, config); write_next_test_port(next_port)?; - if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { - drop(listener); - return Ok(port); + match TcpListener::bind(("127.0.0.1", port)) { + Ok(listener) => { + drop(listener); + return Ok(port); + } + Err(err) => last_error = Some(err), } } - Err("no available E2E test port found".into()) + Err(no_available_test_port_error(config, config.range, last_error.as_ref()).into()) } /// Kill any existing RustFS processes @@ -2201,6 +2214,19 @@ mod tests { assert!(parse_test_port_allocator_config(Some("not-a-port"), Some("128")).is_err()); } + #[test] + fn e2e_port_allocator_reports_attempt_window_and_last_bind_error() { + let config = TestPortAllocatorConfig { min: 41000, range: 3 }; + let error = std::io::Error::from(ErrorKind::PermissionDenied); + + let message = no_available_test_port_error(config, config.range, Some(&error)); + + assert!(message.contains("41000..=41002")); + assert!(message.contains("after 3 attempts")); + assert!(message.contains("last bind error")); + assert!(message.contains("permission denied")); + } + #[test] fn resolves_rustfs_binary_in_configured_cargo_target_directory() { let workspace = Path::new("workspace"); diff --git a/crates/e2e_test/src/distributed/harness.rs b/crates/e2e_test/src/distributed/harness.rs index f610beaa4..bf53789c9 100644 --- a/crates/e2e_test/src/distributed/harness.rs +++ b/crates/e2e_test/src/distributed/harness.rs @@ -59,6 +59,9 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [ #[derive(Clone, Copy, Debug)] pub(crate) enum DistLayout { + /// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the + /// release-evidence EC8+4 geometry. + ThreeByFourEc84, /// 4 nodes × 4 drives, one erasure pool spanning every endpoint. FourByFour, /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout). @@ -94,6 +97,7 @@ impl DistCluster { pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult { let topology = match layout { + DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE), DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE), DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT), DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]), @@ -101,7 +105,7 @@ impl DistCluster { let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?; let pool_storage_roots = match layout { DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?), - DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None, + DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None, }; let mut owned_pool_dirs = Vec::new(); if let Some(roots) = pool_storage_roots.as_deref() { @@ -958,27 +962,6 @@ pub(crate) async fn wait_for_rebalance_active( } } -pub(crate) async fn wait_for_rebalance_running_with_progress( - cluster: &RustFSTestClusterEnvironment, - expected_id: &str, - timeout: Duration, -) -> TestResult { - let deadline = Instant::now() + timeout; - loop { - let status = rebalance_status_json(cluster).await?; - if rebalance_running_with_progress(&status, expected_id)? { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(format!( - "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}" - ) - .into()); - } - sleep(Duration::from_millis(100)).await; - } -} - pub(crate) async fn wait_for_rebalance_complete( cluster: &RustFSTestClusterEnvironment, expected_id: &str, diff --git a/crates/e2e_test/src/distributed/heal_test.rs b/crates/e2e_test/src/distributed/heal_test.rs new file mode 100644 index 000000000..d09c0a32e --- /dev/null +++ b/crates/e2e_test/src/distributed/heal_test.rs @@ -0,0 +1,413 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_inventory, get_object_bytes, payload_for, put_object, sha256_hex, unique_bucket, + wait_until, +}; +use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post}; +use crate::common::init_logging; +use aws_sdk_s3::Client; +use aws_sdk_s3::primitives::ByteStream; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashSet}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tokio::time::{Instant, sleep}; + +const EC84_NODE_COUNT: usize = 3; +const EC84_DRIVES_PER_NODE: usize = 4; +const EC84_DATA_BLOCKS: usize = 8; +const EC84_PARITY_BLOCKS: usize = 4; +const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart"; +const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json"; +const EC84_HEAL_CONTROL_READY_TIMEOUT: Duration = Duration::from_secs(45); +const EC84_HEAL_CONTROL_RETRY_DELAY: Duration = Duration::from_millis(250); + +#[derive(Clone)] +struct ExpectedShard { + key: String, + body: Vec, + baseline: VersionShardCensus, +} + +struct ScannerHealEvidenceContext { + directory: PathBuf, + run: Value, +} + +struct ScannerHealEvidencePayload<'a> { + dist: &'a DistCluster, + bucket: &'a str, + expected: &'a [ExpectedShard], + outage_key: &'a str, + outage_body: &'a [u8], + replaced_drive: &'a Path, + pid_before: u32, + pid_after: u32, + node_listings: Vec>, +} + +fn file_sha256(path: &Path) -> TestResult { + let mut file = std::fs::File::open(path)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect()) +} + +fn compiled_test_identity() -> Value { + serde_json::json!({ + "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"), + "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false", + "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"), + "features": env!("RUSTFS_E2E_BUILD_FEATURES"), + "target": env!("RUSTFS_E2E_BUILD_TARGET"), + "profile": env!("RUSTFS_E2E_BUILD_PROFILE"), + "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"), + }) +} + +fn string_field<'a>(value: &'a Value, path: &str) -> TestResult<&'a str> { + let mut current = value; + for segment in path.split('.') { + current = current + .get(segment) + .ok_or_else(|| format!("scanner/heal run receipt missing {path}"))?; + } + current + .as_str() + .filter(|text| !text.is_empty()) + .ok_or_else(|| format!("scanner/heal run receipt has invalid {path}").into()) +} + +fn scanner_heal_evidence_context() -> TestResult> { + let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else { + return Ok(None); + }; + let directory = PathBuf::from(directory); + let receipt = directory.join("run.json"); + if receipt.metadata()?.len() > 1024 * 1024 { + return Err("oversized scanner/heal execution receipt".into()); + } + let run: Value = serde_json::from_slice(&std::fs::read(receipt)?)?; + let built = compiled_test_identity(); + for key in ["source_revision", "dirty", "lock_blob", "features"] { + if built[key] != run["test_build"][key] { + return Err(format!("compiled test identity differs for {key}").into()); + } + } + let binary_path = PathBuf::from(string_field(&run, "binary.path")?); + if file_sha256(&binary_path)? != string_field(&run, "binary.sha256")? { + return Err("server binary must match the run receipt".into()); + } + if file_sha256(&std::env::current_exe()?)? != string_field(&run, "test_binary.sha256")? { + return Err("test executable must match the run receipt".into()); + } + if directory.join(EC84_TARGET_DRIVE_RESTART_ORACLE).exists() { + return Err("scanner/heal oracle already exists; create a new execution receipt".into()); + } + Ok(Some(ScannerHealEvidenceContext { directory, run })) +} + +fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult { + if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) { + return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into()); + } + let erasure_index = census + .erasure_index + .ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?; + if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) { + return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into()); + } + if !census.is_complete() || census.expected_part_numbers.is_empty() { + return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into()); + } + Ok(()) +} + +async fn write_scanner_heal_evidence(context: ScannerHealEvidenceContext, payload: ScannerHealEvidencePayload<'_>) -> TestResult { + let verifier = payload.dist.client(0)?; + let mut objects = Vec::new(); + for item in payload.expected { + let actual = get_object_bytes(&verifier, payload.bucket, &item.key).await?; + let physical = census_object_version_on_disk(payload.replaced_drive, payload.bucket, &item.key, None)?; + objects.push(serde_json::json!({ + "key": item.key, + "version_id": null, + "expected_bytes": item.body.len(), + "actual_bytes": actual.len(), + "expected_sha256": sha256_hex(&item.body), + "actual_sha256": sha256_hex(&actual), + "expected_physical": item.baseline, + "physical": physical, + })); + } + let actual = get_object_bytes(&verifier, payload.bucket, payload.outage_key).await?; + let physical = census_object_version_on_disk(payload.replaced_drive, payload.bucket, payload.outage_key, None)?; + objects.push(serde_json::json!({ + "key": payload.outage_key, + "version_id": null, + "expected_bytes": payload.outage_body.len(), + "actual_bytes": actual.len(), + "expected_sha256": sha256_hex(payload.outage_body), + "actual_sha256": sha256_hex(&actual), + "expected_physical": null, + "physical": physical, + })); + + let evidence = serde_json::json!({ + "schema": 1, + "case": EC84_TARGET_DRIVE_RESTART_CASE, + "evidence": "process-restart", + "run_id": string_field(&context.run, "run_id")?, + "source_revision": string_field(&context.run, "source_revision")?, + "test_build": compiled_test_identity(), + "binary_sha256": string_field(&context.run, "binary.sha256")?, + "test_binary_sha256": string_field(&context.run, "test_binary.sha256")?, + "topology": {"nodes": EC84_NODE_COUNT, "drives_per_node": EC84_DRIVES_PER_NODE}, + "pid_before": payload.pid_before, + "pid_after": payload.pid_after, + "unclean_shutdown_marker": false, + "objects": objects, + "node_listings": payload.node_listings, + }); + let data = serde_json::to_vec(&evidence)?; + if data.len() > 1024 * 1024 { + return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into()); + } + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(context.directory.join(EC84_TARGET_DRIVE_RESTART_ORACLE))?; + output.write_all(&data)?; + output.sync_all()?; + Ok(()) +} + +fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult { + for key in keys { + let census = census_object_version_on_disk(drive, bucket, key, None)?; + if census.has_xl_meta { + return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into()); + } + } + Ok(()) +} + +fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool { + let message = error.to_string(); + message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable") +} + +async fn start_ec84_root_heal_when_control_ready( + heal_url: &str, + heal_body: &str, + access_key: &str, + secret_key: &str, +) -> TestResult { + let deadline = Instant::now() + EC84_HEAL_CONTROL_READY_TIMEOUT; + loop { + match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await { + Ok(_) => return Ok(()), + Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && Instant::now() < deadline => { + sleep(EC84_HEAL_CONTROL_RETRY_DELAY).await; + } + Err(error) => return Err(error), + } + } +} + +async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult> { + let mut expected = Vec::new(); + for index in 0..4 { + let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2); + let body = payload_for(&key, 10 * 1024 * 1024); + put_object(client, bucket, &key, body.clone()).await?; + expected.push(ExpectedShard { + key, + body, + baseline: VersionShardCensus { + version_id: None, + has_xl_meta: false, + data_dir: None, + erasure_index: None, + data_blocks: None, + parity_blocks: None, + expected_part_numbers: Default::default(), + present_part_fingerprints: Default::default(), + inline_data_fingerprint: None, + }, + }); + } + Ok(expected) +} + +#[tokio::test] +async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult { + init_logging(); + let evidence_context = scanner_heal_evidence_context()?; + let mut dist = DistCluster::start_with_env( + DistLayout::ThreeByFourEc84, + &[ + ("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"), + ("RUSTFS_HEAL_ENABLED", "true"), + ("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"), + ("RUSTFS_HEAL_MRF_ENABLE", "false"), + ("RUSTFS_SCANNER_ENABLED", "false"), + ], + ) + .await?; + assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT); + assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE); + + let bucket = unique_bucket("healec84"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let mut expected = put_large_inventory(&writer, &bucket).await?; + let replaced_node = 1; + let replaced_drive_index = 2; + let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]); + + for item in &mut expected { + item.baseline = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?; + assert_ec84_geometry(&item.baseline, &item.key)?; + } + + let format_path = replaced_drive.join(".rustfs.sys").join("format.json"); + let format_json = std::fs::read(&format_path)?; + let target_pid_before = dist.cluster.nodes[replaced_node] + .process + .as_ref() + .ok_or("target process is absent before graceful restart")? + .id(); + dist.cluster.stop_node_gracefully(replaced_node).await?; + let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display())); + std::fs::rename(&replaced_drive, &retired_drive)?; + std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?; + std::fs::write(&format_path, format_json)?; + assert_replaced_drive_empty( + &replaced_drive, + &bucket, + &expected.iter().map(|item| item.key.clone()).collect::>(), + )?; + + let outage_key = "ec84/written-while-node-restarting.bin"; + let outage_body = payload_for(outage_key, 10 * 1024 * 1024); + writer + .put_object() + .bucket(&bucket) + .key(outage_key) + .body(ByteStream::from(outage_body.clone())) + .send() + .await?; + + dist.cluster.start_node(replaced_node).await?; + let target_pid_after = dist.cluster.nodes[replaced_node] + .process + .as_ref() + .ok_or("target process is absent after restart")? + .id(); + let heal_body = + r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; + let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url); + start_ec84_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?; + + wait_until( + Duration::from_secs(120), + || async { + for item in &expected { + let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?; + if !current.matches_manifest(&item.baseline) { + return Ok(false); + } + } + let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?; + Ok(outage.is_complete() + && outage.data_blocks == Some(EC84_DATA_BLOCKS) + && outage.parity_blocks == Some(EC84_PARITY_BLOCKS)) + }, + "EC8+4 replacement drive rebuilt baseline and outage shards", + ) + .await?; + + let inventory = expected + .iter() + .map(|item| (item.key.clone(), item.body.clone())) + .chain(std::iter::once((outage_key.to_string(), outage_body.clone()))) + .collect::>(); + let expected_keys = inventory.keys().cloned().collect::>(); + let mut node_listings = Vec::new(); + for node_index in 0..dist.cluster.nodes.len() { + let client = dist.client(node_index)?; + assert_inventory(&client, &bucket, &inventory).await?; + let listing = client.list_objects_v2().bucket(&bucket).send().await?; + let observed = listing + .contents() + .iter() + .filter_map(|object| object.key().map(str::to_owned)) + .collect::>(); + assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal"); + let mut observed = observed.into_iter().collect::>(); + observed.sort(); + node_listings.push(observed); + } + if let Some(context) = evidence_context { + write_scanner_heal_evidence( + context, + ScannerHealEvidencePayload { + dist: &dist, + bucket: &bucket, + expected: &expected, + outage_key, + outage_body: &outage_body, + replaced_drive: &replaced_drive, + pid_before: target_pid_before, + pid_after: target_pid_after, + node_listings, + }, + ) + .await?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cluster_heal_coordination_retry_is_exact() { + let retryable: Box = + "admin POST failed: 500 Internal Server Error cluster heal coordination unavailable".into(); + assert!(is_cluster_heal_coordination_unavailable(retryable.as_ref())); + + let other_internal: Box = + "admin POST failed: 500 Internal Server Error unrelated".into(); + assert!(!is_cluster_heal_coordination_unavailable(other_internal.as_ref())); + + let wrong_status: Box = + "admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into(); + assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref())); + } +} diff --git a/crates/e2e_test/src/distributed/mod.rs b/crates/e2e_test/src/distributed/mod.rs index 1759440f8..0a3b0a086 100644 --- a/crates/e2e_test/src/distributed/mod.rs +++ b/crates/e2e_test/src/distributed/mod.rs @@ -25,6 +25,7 @@ mod data_integrity_movement_test; mod expand_decommission_rebalance_test; mod extra_test; mod harness; +mod heal_test; mod object_lock_test; mod observability_test; mod replication_delete_marker_test; diff --git a/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs index 66c835bbd..79b7710d1 100644 --- a/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs +++ b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs @@ -14,9 +14,9 @@ use super::harness::{ DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, - decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json, - retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete, - wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress, + decommission_status_json, put_inventory_retrying, rebalance_active, rebalance_status_json, retrying_get_equals, retrying_put, + start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete, + wait_for_decommission_running_with_progress, wait_for_rebalance_active, wait_for_rebalance_complete, }; use crate::common::init_logging; use std::time::Duration; @@ -67,7 +67,10 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu assert_inventory(&live, &bucket, &inventory).await?; let rebalance_id = start_rebalance(&dist.cluster).await?; - wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; + // The status API reads persisted progress, whose first periodic save is + // after 30 seconds. A shorter run can remain at zero until completion. + // Require Started around the S3 operations and nonzero progress at completion. + wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; retrying_put( &live, &bucket, @@ -84,11 +87,26 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu Duration::from_secs(30), ) .await?; + let listed = live.list_objects_v2().bucket(&bucket).send().await?; + assert!( + listed + .contents() + .iter() + .any(|object| object.key() == Some("during-rebalance.bin")), + "list during rebalance missed the newly written key" + ); let status = rebalance_status_json(&dist.cluster).await?; - if !rebalance_running_with_progress(&status, &rebalance_id)? { + if !rebalance_active(&status, &rebalance_id)? { return Err(format!("rebalance did not remain active across the S3 operations: {status}").into()); } wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?; - assert_inventory(&dist.client(1)?, &bucket, &inventory).await?; + let after = dist.client(1)?; + assert_inventory(&after, &bucket, &inventory).await?; + for (key, body) in [ + ("during-decommission.bin", b"written-while-decommissioning".as_slice()), + ("during-rebalance.bin", b"written-while-rebalancing".as_slice()), + ] { + retrying_get_equals(&after, &bucket, key, body, Duration::from_secs(30)).await?; + } Ok(()) } diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index a2bb5c425..415673a17 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -18,11 +18,15 @@ mod tests { use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post}; use crate::common::{ - FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, - rustfs_binary_path, + ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, + init_logging, rustfs_binary_path, }; use crate::storage_api::RUSTFS_META_BUCKET; - use aws_sdk_s3::primitives::ByteStream; + use aws_sdk_s3::{ + error::{ProvideErrorMetadata, SdkError}, + operation::put_object::PutObjectError, + primitives::ByteStream, + }; use http::Method; use sha2::{Digest, Sha256}; use std::collections::HashSet; @@ -39,6 +43,13 @@ mod tests { const POOL_METADATA_OBJECT: &str = "pool.bin"; + struct ReplacementDriveSelection { + replaced_disk: PathBuf, + replacement_format_path: PathBuf, + replacement_format: Vec, + expected_pool_metadata: Option, + } + #[derive(serde::Deserialize)] struct EvidenceBuild { sha256: String, @@ -60,6 +71,65 @@ mod tests { oracle: &'static str, evidence: &'static str, unclean_shutdown_marker: bool, + topology: EvidenceTopology, + storage_class_standard: Option<&'static str>, + erasure_set_drive_count: Option, + outage_target_manifest_required: bool, + } + + #[derive(Clone, Copy)] + struct EvidenceTopology { + nodes: usize, + drives_per_node: usize, + layout: EvidenceTopologyLayout, + } + + #[derive(Clone, Copy)] + enum EvidenceTopologyLayout { + SinglePool, + PerNodePools, + } + + impl EvidenceTopology { + const fn new(nodes: usize, drives_per_node: usize) -> Self { + Self { + nodes, + drives_per_node, + layout: EvidenceTopologyLayout::SinglePool, + } + } + + const fn per_node_pools(nodes: usize, drives_per_node: usize) -> Self { + Self { + nodes, + drives_per_node, + layout: EvidenceTopologyLayout::PerNodePools, + } + } + + fn total_drives(self) -> usize { + self.nodes * self.drives_per_node + } + + fn cluster_topology(self) -> ClusterTopology { + match self.layout { + EvidenceTopologyLayout::SinglePool => ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node), + EvidenceTopologyLayout::PerNodePools => { + ClusterTopology::per_node_pools(self.drives_per_node, (0..self.nodes).map(|node| vec![node]).collect()) + } + } + } + + fn pool_count(self) -> usize { + match self.layout { + EvidenceTopologyLayout::SinglePool => 1, + EvidenceTopologyLayout::PerNodePools => self.nodes, + } + } + + fn set_count(self, erasure_set_drive_count: Option) -> usize { + self.total_drives() / erasure_set_drive_count.unwrap_or_else(|| self.total_drives()) + } } const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { @@ -67,6 +137,10 @@ mod tests { oracle: "background-target-restart.json", evidence: "process-restart", unclean_shutdown_marker: false, + topology: EvidenceTopology::new(4, 1), + storage_class_standard: None, + erasure_set_drive_count: None, + outage_target_manifest_required: true, }; const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { @@ -74,6 +148,54 @@ mod tests { oracle: "background-target-crash.json", evidence: "process-crash-restart", unclean_shutdown_marker: true, + topology: EvidenceTopology::new(4, 1), + storage_class_standard: None, + erasure_set_drive_count: None, + outage_target_manifest_required: true, + }; + + const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { + id: "background-target-restart-ec8-4", + oracle: "background-target-restart-ec8-4.json", + evidence: "process-restart", + unclean_shutdown_marker: false, + topology: EvidenceTopology::new(3, 4), + storage_class_standard: Some("EC:4"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: true, + }; + + const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { + id: "background-target-crash-ec8-4", + oracle: "background-target-crash-ec8-4.json", + evidence: "process-crash-restart", + unclean_shutdown_marker: true, + topology: EvidenceTopology::new(3, 4), + storage_class_standard: Some("EC:4"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: true, + }; + + const BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { + id: "background-target-restart-ec8-4-multi-set", + oracle: "background-target-restart-ec8-4-multi-set.json", + evidence: "process-restart", + unclean_shutdown_marker: false, + topology: EvidenceTopology::new(3, 8), + storage_class_standard: Some("EC:4"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: true, + }; + + const BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase { + id: "background-target-crash-ec8-4-multi-pool", + oracle: "background-target-crash-ec8-4-multi-pool.json", + evidence: "process-crash-restart", + unclean_shutdown_marker: true, + topology: EvidenceTopology::per_node_pools(3, 12), + storage_class_standard: Some("EC:4"), + erasure_set_drive_count: Some(12), + outage_target_manifest_required: false, }; struct RestartEvidenceContext { @@ -332,11 +454,10 @@ mod tests { // Healing may rewrite non-identity bookkeeping in xl.meta. The census // therefore compares the canonical selected metadata fields plus every - // physical shard, while the payload seed makes object mix-ups observable. + // physical shard. #[derive(Debug)] struct PhysicalObjectManifest { key: String, - payload_seed: u8, shard_census: VersionShardCensus, } @@ -476,6 +597,54 @@ mod tests { && operations["activeBySource"]["admin"].as_u64() == Some(1) } + fn is_service_unavailable_put(error: &SdkError) -> bool { + error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable") + } + + fn select_replacement_drive( + cluster: &RustFSTestClusterEnvironment, + node_index: usize, + require_pool_metadata: bool, + ) -> Result> { + let node = cluster + .nodes + .get(node_index) + .ok_or_else(|| format!("replacement node {node_index} is absent"))?; + let mut incomplete_pool_metadata = Vec::new(); + + for drive in &node.data_dirs { + let replaced_disk = PathBuf::from(drive); + let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json"); + let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| { + format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}") + })?; + if !require_pool_metadata { + return Ok(ReplacementDriveSelection { + replaced_disk, + replacement_format_path, + replacement_format, + expected_pool_metadata: None, + }); + } + + let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?; + if census.is_complete() { + return Ok(ReplacementDriveSelection { + replaced_disk, + replacement_format_path, + replacement_format, + expected_pool_metadata: Some(census), + }); + } + incomplete_pool_metadata.push(census); + } + + Err(format!( + "no replacement drive on node {node_index} held complete pool metadata before the fault: {incomplete_pool_metadata:?}" + ) + .into()) + } + async fn replacement_recovery_status( cluster: &RustFSTestClusterEnvironment, ) -> Result> { @@ -898,7 +1067,7 @@ mod tests { let mut versions_observed = false; let mut observations = Vec::with_capacity(cluster.nodes.len()); for (node_index, node) in cluster.nodes.iter().enumerate() { - let (status, body) = timeout( + let response = timeout( Duration::from_secs(5), admin_request( &node.url, @@ -909,8 +1078,22 @@ mod tests { &cluster.secret_key, ), ) - .await??; - assert_eq!(status, 200, "scanner status must be available: {body}"); + .await; + let (status, body) = match response { + Ok(Ok(response)) => response, + Ok(Err(error)) => { + observations.push(format!("node{node_index}: scanner status request failed: {error}")); + continue; + } + Err(_) => { + observations.push(format!("node{node_index}: scanner status request exceeded 5s")); + continue; + } + }; + if status != 200 { + observations.push(format!("node{node_index}: scanner status returned {status}: {body}")); + continue; + } let status: serde_json::Value = serde_json::from_str(&body)?; assert_eq!(status["enabled"].as_bool(), Some(true), "scanner must stay enabled: {status}"); let metrics = &status["metrics"]; @@ -973,6 +1156,46 @@ mod tests { .await? } + #[tokio::test(flavor = "multi_thread")] + async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart() + -> Result<(), Box> { + timeout( + Duration::from_secs(420), + run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84), + ) + .await? + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash() + -> Result<(), Box> { + timeout( + Duration::from_secs(420), + run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84), + ) + .await? + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart() + -> Result<(), Box> { + timeout( + Duration::from_secs(600), + run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet), + ) + .await? + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash() + -> Result<(), Box> { + timeout( + Duration::from_secs(600), + run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84MultiPool), + ) + .await? + } + #[tokio::test(flavor = "multi_thread")] async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box> { @@ -1010,29 +1233,48 @@ mod tests { IsolatedTargetRestart, BackgroundTargetRestart, BackgroundTargetCrash, + BackgroundTargetRestartEc84, + BackgroundTargetCrashEc84, + BackgroundTargetRestartEc84MultiSet, + BackgroundTargetCrashEc84MultiPool, BackgroundCoordinatorRestart, TargetEndpointBlackhole, } async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box> { let server_binary = rustfs_binary_path(); - let evidence_run = match scenario { - InterruptionScenario::BackgroundTargetRestart => { - restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)? - } - InterruptionScenario::BackgroundTargetCrash => { - restart_evidence_run(&server_binary, BACKGROUND_TARGET_CRASH_EVIDENCE)? - } + let evidence_case = match scenario { + InterruptionScenario::BackgroundTargetRestart => Some(BACKGROUND_TARGET_RESTART_EVIDENCE), + InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE), + InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE), + InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE), + InterruptionScenario::BackgroundTargetRestartEc84MultiSet => Some(BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE), + InterruptionScenario::BackgroundTargetCrashEc84MultiPool => Some(BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE), _ => None, }; + let evidence_run = match evidence_case { + Some(case) => restart_evidence_run(&server_binary, case)?, + None => None, + }; let mut evidence_objects = Vec::new(); let (background_enabled, interruption_node, interruption_kind) = match scenario { InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"), InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"), InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"), + InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"), + InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"), + InterruptionScenario::BackgroundTargetRestartEc84MultiSet => (true, 1, "background_target_restart_ec8_4_multi_set"), + InterruptionScenario::BackgroundTargetCrashEc84MultiPool => (true, 1, "background_target_crash_ec8_4_multi_pool"), InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"), InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"), }; + let topology = evidence_case + .map(|case| case.topology) + .unwrap_or_else(|| EvidenceTopology::new(4, 1)); + let erasure_set_drive_count = evidence_case + .and_then(|case| case.erasure_set_drive_count) + .unwrap_or_else(|| topology.total_drives()); + let outage_target_manifest_required = evidence_case.map(|case| case.outage_target_manifest_required).unwrap_or(true); init_logging(); info!( event = "heal_interruption_started", @@ -1044,9 +1286,15 @@ mod tests { "Starting root-heal interruption test" ); - let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?; cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true"); cluster.set_env("RUSTFS_HEAL_ENABLED", "true"); + if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) { + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class); + } + if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) { + cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count.to_string()); + } // Capture physical baselines after the PUT rename fanout has drained. cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); // Heal control uses the first lexicographically sorted grid host. @@ -1082,26 +1330,39 @@ mod tests { let bucket = "heal-restart-during-rebuild"; clients[0].create_bucket().bucket(bucket).send().await?; - let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir); - let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json"); - let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| { - format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}") - })?; + let ReplacementDriveSelection { + replaced_disk, + replacement_format_path, + replacement_format, + expected_pool_metadata, + } = select_replacement_drive(&cluster, 1, background_enabled)?; + let default_online_object_count = if !outage_target_manifest_required { 64 } else { 24 }; let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT") .ok() .and_then(|value| value.parse::().ok()) - .unwrap_or(24) - .clamp(8, 64); + .unwrap_or(default_online_object_count) + .clamp(8, 128); let object_size_bytes = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES") .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(4 * 1024 * 1024) .clamp(1024 * 1024, 16 * 1024 * 1024); + let mut created_online_objects = Vec::with_capacity(online_object_count); let mut expected_manifests = Vec::with_capacity(online_object_count); let mut unclean_shutdown_marker_observed = None; - for index in 0..online_object_count { - let key = format!("cluster/online/object-{index:04}.bin"); - let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8"); + let mut attempt_count = 0usize; + let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1)); + while expected_manifests.len() < online_object_count { + if attempt_count >= max_online_attempts { + return Err(format!( + "target replacement drive held only {}/{} baseline object shards after {attempt_count} writes", + expected_manifests.len(), + online_object_count + ) + .into()); + } + let key = format!("cluster/online/object-{attempt_count:04}.bin"); + let payload_seed = ((attempt_count % 251) + 1) as u8; timeout( Duration::from_secs(30), clients[0] @@ -1113,6 +1374,11 @@ mod tests { ) .await??; let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?; + if !shard_census.has_xl_meta { + timeout(Duration::from_secs(30), clients[0].delete_object().bucket(bucket).key(&key).send()).await??; + attempt_count += 1; + continue; + } assert!( shard_census.is_complete(), "node 1 should hold a complete baseline shard for {key}: {shard_census:?}" @@ -1121,24 +1387,14 @@ mod tests { !shard_census.expected_part_numbers.is_empty(), "chaos objects must use physical part shards rather than inline data: {shard_census:?}" ); - expected_manifests.push(PhysicalObjectManifest { - key, - payload_seed, - shard_census, - }); + created_online_objects.push((key.clone(), payload_seed)); + expected_manifests.push(PhysicalObjectManifest { key, shard_census }); + attempt_count += 1; } - let expected_pool_metadata = if background_enabled { - let census = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?; - assert!( - census.is_complete(), - "target must hold complete pool metadata before the fault: {census:?}" - ); + if background_enabled { wait_for_scanner_cycle_after(&cluster, 0).await?; - Some(census) - } else { - None - }; + } cluster.stop_node(1)?; std::fs::remove_dir_all(&replaced_disk)?; @@ -1153,49 +1409,105 @@ mod tests { "replacement target must retain only its preformatted topology identity" ); - let outage_key = "cluster/written-while-node-down.bin"; let outage_payload_seed = 0xf1; - timeout( - Duration::from_secs(30), - clients[2] - .put_object() - .bucket(bucket) - .key(outage_key) - .body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed))) - .send(), - ) - .await??; + let max_outage_write_attempts = topology.total_drives().max(1); + let mut outage_key = None; + let mut outage_write_deferred_until_rejoin = false; + let mut service_unavailable_outage_writes = 0usize; + let mut last_service_unavailable = None; + for attempt in 0..max_outage_write_attempts { + let candidate_key = format!("cluster/written-while-node-down-{attempt:04}.bin"); + let put_result = timeout( + Duration::from_secs(30), + clients[2] + .put_object() + .bucket(bucket) + .key(&candidate_key) + .body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed))) + .send(), + ) + .await; + match put_result { + Ok(Ok(_)) => { + outage_key = Some(candidate_key); + break; + } + Ok(Err(error)) if is_service_unavailable_put(&error) => { + service_unavailable_outage_writes += 1; + last_service_unavailable = Some(format!("{error:?}")); + } + Ok(Err(error)) => return Err(error.into()), + Err(error) => return Err(error.into()), + } + } + let outage_key = match outage_key { + Some(key) => key, + None if !outage_target_manifest_required => { + outage_write_deferred_until_rejoin = true; + "cluster/written-after-target-rejoin.bin".to_string() + } + None => { + return Err(format!( + "no online pool accepted an outage object after {max_outage_write_attempts} candidates; \ + observed {service_unavailable_outage_writes} ServiceUnavailable responses; \ + last ServiceUnavailable: {last_service_unavailable:?}" + ) + .into()); + } + }; let mut outage_peer_erasure_indices = HashSet::new(); - for (node_index, node) in cluster.nodes.iter().enumerate() { - if node_index == 1 { - continue; + if !outage_write_deferred_until_rejoin { + for (node_index, node) in cluster.nodes.iter().enumerate() { + if node_index == 1 { + continue; + } + for (drive_index, drive) in node.data_dirs.iter().enumerate() { + let census = census_object_version_on_disk(Path::new(drive), bucket, &outage_key, None)?; + if !census.has_xl_meta { + continue; + } + assert!( + census.is_complete(), + "online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}" + ); + let erasure_index = census.erasure_index.ok_or_else(|| { + format!( + "online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}" + ) + })?; + assert!( + (1..=erasure_set_drive_count).contains(&erasure_index), + "online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}" + ); + assert!( + outage_peer_erasure_indices.insert(erasure_index), + "outage-object erasure index {erasure_index} is duplicated across online drives" + ); + } } - let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?; + } + assert!( + outage_write_deferred_until_rejoin + || (!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count), + "outage-object must occupy one non-empty erasure set" + ); + if outage_target_manifest_required { + let min_online_data_shards = erasure_set_drive_count.saturating_sub(4); assert!( - census.is_complete(), - "online node {node_index} must hold a complete outage-object shard: {census:?}" + outage_peer_erasure_indices.len() >= min_online_data_shards, + "online drives in the selected erasure set must retain at least the EC data quorum" ); - let erasure_index = census - .erasure_index - .ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?; + } + let missing_outage_erasure_indices = (1..=erasure_set_drive_count) + .filter(|index| !outage_peer_erasure_indices.contains(index)) + .collect::>(); + if outage_target_manifest_required { assert!( - (1..=cluster.nodes.len()).contains(&erasure_index), - "online node {node_index} outage-object erasure index is out of range: {census:?}" - ); - assert!( - outage_peer_erasure_indices.insert(erasure_index), - "outage-object erasure index {erasure_index} is duplicated across online nodes" + !missing_outage_erasure_indices.is_empty(), + "the stopped target must account for at least one missing outage-object erasure index" ); } - assert_eq!( - outage_peer_erasure_indices.len(), - cluster.nodes.len().saturating_sub(1), - "every online node must contribute one unique outage-object erasure index" - ); - let expected_outage_target_erasure_index = (1..=cluster.nodes.len()) - .find(|index| !outage_peer_erasure_indices.contains(index)) - .ok_or("online outage-object shards leave no erasure index for the replacement target")?; let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; if !background_enabled { @@ -1230,7 +1542,12 @@ mod tests { ); let recovered: serde_json::Value = serde_json::from_str(&status_body) .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?; - let ready = if background_enabled { + let ready = if background_enabled && outage_write_deferred_until_rejoin { + // Whole-pool outage writes are deferred, so start the admin + // root heal as soon as the target answers instead of waiting + // for background convergence to consume the interruption window. + true + } else if background_enabled { recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) } else { cluster_heal_is_idle(&recovered) @@ -1251,7 +1568,7 @@ mod tests { "non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal" ); assert!( - !census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta, + !census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?.has_xl_meta, "the object written during the outage must be absent before the explicit root heal" ); assert_eq!( @@ -1261,38 +1578,50 @@ mod tests { ); } + let background_rejoin_heal_evidence = background_enabled && outage_write_deferred_until_rejoin; let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url); - let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?; - let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body) - .map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?; - let client_token = heal_start["clientToken"] - .as_str() - .filter(|token| !token.is_empty()) - .ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?; - let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url); + let (mut client_token, mut task_status_url) = if background_rejoin_heal_evidence { + (String::new(), String::new()) + } else { + let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?; + let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body) + .map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?; + let client_token = heal_start["clientToken"] + .as_str() + .filter(|token| !token.is_empty()) + .ok_or_else(|| format!("heal start response has no client token: {heal_start}"))? + .to_string(); + let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url); + (client_token, task_status_url) + }; + let restart_recovery_admin_after_failure = + scenario == InterruptionScenario::BackgroundTargetCrashEc84MultiPool && !background_rejoin_heal_evidence; + let mut recovery_admin_task_restarted = false; let partial_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS") .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(60); let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs); - loop { - let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?; - let active_status: serde_json::Value = serde_json::from_str(&status_body) - .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?; - let active = if background_enabled { - active_status["state"].as_str() == Some("active") - && active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1) - } else { - only_admin_heal_is_active(&active_status) - }; - if active { - break; + if !background_rejoin_heal_evidence { + loop { + let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?; + let active_status: serde_json::Value = serde_json::from_str(&status_body) + .map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?; + let active = if background_enabled { + active_status["state"].as_str() == Some("active") + && active_status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1) + } else { + only_admin_heal_is_active(&active_status) + }; + if active { + break; + } + if Instant::now() >= partial_deadline { + return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into()); + } + sleep(Duration::from_millis(50)).await; } - if Instant::now() >= partial_deadline { - return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into()); - } - sleep(Duration::from_millis(50)).await; } let (partial_count, partial_manifest) = loop { // Hash one committed shard to prove progress without letting a @@ -1310,14 +1639,24 @@ mod tests { } if materialized == expected_manifests.len() { return Err(format!( - "root heal rebuilt all {} baseline objects before the target could be interrupted", + "{} rebuilt all {} baseline objects before the target could be interrupted", + if background_rejoin_heal_evidence { + "background rejoin heal" + } else { + "root heal" + }, expected_manifests.len() ) .into()); } if Instant::now() >= partial_deadline { return Err(format!( - "root heal made no observable partial progress on the replacement target within {partial_timeout_secs}s" + "{} made no observable partial progress on the replacement target within {partial_timeout_secs}s", + if background_rejoin_heal_evidence { + "background rejoin heal" + } else { + "root heal" + } ) .into()); } @@ -1328,25 +1667,40 @@ mod tests { let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body) .map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?; let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?; - let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?; - assert!( - coordinator_log - .lines() - .filter_map(|line| serde_json::from_str::(line).ok()) - .any(|event| { - event["event"] == "heal_task_state" - && event["task_id"] == client_token - && event["heal_type"] == "cluster" - && event["state"] == "started" - }), - "node 0 must have started the exact admin task before interruption" - ); + if background_rejoin_heal_evidence { + let target_log = std::fs::read_to_string(format!("{log_dir}/node1.log"))?; + assert!( + target_log + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .any(|event| { + event["event"] == "heal_task_state" && event["heal_type"] == "erasure_set" && event["state"] == "started" + }), + "node 1 must have started a background erasure-set heal before interruption" + ); + } else { + let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?; + assert!( + coordinator_log + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .any(|event| { + event["event"] == "heal_task_state" + && event["task_id"].as_str() == Some(client_token.as_str()) + && event["heal_type"] == "cluster" + && event["state"] == "started" + }), + "node 0 must have started the exact admin task before interruption" + ); + } let pre_interrupt_operations = &pre_interrupt_status["healOperations"]; - assert_eq!( - pre_interrupt_operations["activeBySource"]["admin"].as_u64(), - Some(1), - "interruption must occur while the single admin task is active: {pre_interrupt_status}" - ); + if !background_rejoin_heal_evidence { + assert_eq!( + pre_interrupt_operations["activeBySource"]["admin"].as_u64(), + Some(1), + "interruption must occur while the single admin task is active: {pre_interrupt_status}" + ); + } if !background_enabled { assert!( only_admin_heal_is_active(&pre_interrupt_status), @@ -1465,7 +1819,13 @@ mod tests { "Restored target endpoint forwarding" ); } else { - if scenario == InterruptionScenario::BackgroundTargetRestart { + if matches!( + scenario, + InterruptionScenario::BackgroundTargetRestart + | InterruptionScenario::BackgroundTargetRestartEc84 + | InterruptionScenario::BackgroundTargetRestartEc84MultiSet + | InterruptionScenario::BackgroundCoordinatorRestart + ) { cluster.stop_node_gracefully(interruption_node).await?; } else { cluster.stop_node(interruption_node)?; @@ -1487,7 +1847,12 @@ mod tests { if background_enabled { let marker_exists = unclean_shutdown_marker.is_file(); unclean_shutdown_marker_observed = Some(marker_exists); - let expected_marker = !matches!(scenario, InterruptionScenario::BackgroundTargetRestart); + let expected_marker = matches!( + scenario, + InterruptionScenario::BackgroundTargetCrash + | InterruptionScenario::BackgroundTargetCrashEc84 + | InterruptionScenario::BackgroundTargetCrashEc84MultiPool + ); assert!( marker_exists == expected_marker, "background restart/crash lane observed unexpected unclean-shutdown marker state" @@ -1526,11 +1891,43 @@ mod tests { .unwrap_or(180); let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs); loop { - if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len() - && object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key) - { + if restart_recovery_admin_after_failure && !recovery_admin_task_restarted { + let task_state = timeout( + Duration::from_secs(2), + signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key), + ) + .await; + if let Ok(Ok(body)) = task_state + && let Ok(status) = serde_json::from_str::(&body) + && status["summary"].as_str() == Some("failed") + { + let heal_start_body = + signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?; + let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body) + .map_err(|err| format!("recovery heal start response is not JSON ({err}): {heal_start_body}"))?; + client_token = heal_start["clientToken"] + .as_str() + .filter(|token| !token.is_empty()) + .ok_or_else(|| format!("recovery heal start response has no client token: {heal_start}"))? + .to_string(); + task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url); + recovery_admin_task_restarted = true; + info!( + event = "heal_interruption_recovery_task_restarted", + component = "e2e_test", + subsystem = "heal", + interruption_kind, + recovery_client_token = client_token, + "Restarted admin root heal after interrupted target-crash task failed" + ); + } + } + let baseline_recovered = metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len(); + let outage_recovered = + !outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, &outage_key); + if baseline_recovered && outage_recovered { let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?; - let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?; + let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?; let pool_metadata_matches = match &expected_pool_metadata { Some(expected) => { census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)? @@ -1538,27 +1935,34 @@ mod tests { } None => true, }; - if matching == expected_manifests.len() && outage_census.is_complete() && pool_metadata_matches { + if matching == expected_manifests.len() + && (!outage_target_manifest_required || outage_census.is_complete()) + && pool_metadata_matches + { break; } } if Instant::now() >= heal_deadline { let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?; - let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?; + let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?; let pool_metadata = census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?; let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key) .await .unwrap_or_else(|err| format!("status request failed: {err}")); - let task_status = match timeout( - Duration::from_secs(5), - signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key), - ) - .await - { - Ok(Ok(body)) => heal_task_status_diagnostic(&body), - Ok(Err(err)) => format!("task status request failed: {err}"), - Err(_) => "task status request exceeded 5s diagnostic budget".to_string(), + let task_status = if background_rejoin_heal_evidence { + "background erasure-set heal has no admin root-heal task token".to_string() + } else { + match timeout( + Duration::from_secs(5), + signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key), + ) + .await + { + Ok(Ok(body)) => heal_task_status_diagnostic(&body), + Ok(Err(err)) => format!("task status request failed: {err}"), + Err(_) => "task status request exceeded 5s diagnostic budget".to_string(), + } }; let replacement_status = match timeout(Duration::from_secs(5), replacement_recovery_status(&cluster)).await { Ok(Ok(status)) => status.to_string(), @@ -1582,35 +1986,75 @@ mod tests { expected.key ); } - let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?; - assert!( - outage_census.is_complete(), - "outage object must have a complete target shard: {outage_census:?}" - ); - assert_eq!( - outage_census.erasure_index, - Some(expected_outage_target_erasure_index), - "the outage object must be rebuilt into its own missing erasure slot" - ); + let outage_census = census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?; + if outage_target_manifest_required { + assert!( + outage_census.is_complete(), + "outage object must have a complete target shard: {outage_census:?}" + ); + assert_eq!( + outage_census + .erasure_index + .filter(|index| missing_outage_erasure_indices.contains(index)), + outage_census.erasure_index, + "the outage object must be rebuilt into one of the stopped node's missing erasure slots" + ); + } if let Some(cycle_end) = scanner_cycle_floor { wait_for_scanner_cycle_after(&cluster, cycle_end).await?; } - let mut expected_keys = expected_manifests + if outage_write_deferred_until_rejoin { + let deferred_deadline = Instant::now() + Duration::from_secs(60); + loop { + let put_result = timeout( + Duration::from_secs(30), + clients[0] + .put_object() + .bucket(bucket) + .key(&outage_key) + .body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed))) + .send(), + ) + .await; + match put_result { + Ok(Ok(_)) => break, + Ok(Err(error)) if is_service_unavailable_put(&error) && Instant::now() < deferred_deadline => { + sleep(Duration::from_secs(1)).await; + } + Ok(Err(error)) => return Err(error.into()), + Err(error) => return Err(error.into()), + } + } + info!( + event = "heal_interruption_outage_write_deferred", + component = "e2e_test", + subsystem = "heal", + interruption_kind, + outage_key, + "Deferred whole-pool outage write until the target pool rejoined" + ); + } + + let mut expected_keys = created_online_objects .iter() - .map(|manifest| manifest.key.clone()) + .map(|(key, _)| key.clone()) .collect::>(); - assert!(expected_keys.insert(outage_key.to_string())); + assert!(expected_keys.insert(outage_key.clone())); let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?; let target_client = cluster.create_s3_client(1)?; - for expected in &expected_manifests { - let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?; + for (key, payload_seed) in &created_online_objects { + let response = target_client.get_object().bucket(bucket).key(key).send().await?; let actual = response.body.collect().await?.into_bytes(); - let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed); - assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key); - if evidence_run.is_some() { + let expected_body = deterministic_object_body(object_size_bytes, *payload_seed); + assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {key}"); + if evidence_run.is_some() + && let Some(expected) = expected_manifests + .iter() + .find(|manifest| manifest.key.as_str() == key.as_str()) + { evidence_objects.push(serde_json::json!({ "key": expected.key, "version_id": expected.shard_census.version_id, "expected_bytes": expected_body.len(), "actual_bytes": actual.len(), @@ -1621,7 +2065,7 @@ mod tests { })); } } - let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?; + let response = target_client.get_object().bucket(bucket).key(&outage_key).send().await?; let actual = response.body.collect().await?.into_bytes(); let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed); assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}"); @@ -1632,7 +2076,7 @@ mod tests { "expected_sha256": sha256_hex(&expected_outage_body), "actual_sha256": sha256_hex(&actual), "expected_physical": null, - "physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?, + "physical": census_object_version_on_disk(&replaced_disk, bucket, &outage_key, None)?, })); } @@ -1650,36 +2094,27 @@ mod tests { sleep(Duration::from_millis(250)).await; } - let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?; - let task_status: serde_json::Value = serde_json::from_str(&task_status_body) - .map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?; + if !background_rejoin_heal_evidence { + let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?; + let task_status: serde_json::Value = serde_json::from_str(&task_status_body) + .map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?; + if task_status["summary"].as_str() != Some("finished") { + return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into()); + } + } if interruption_node == 0 { - // Admin tasks are process-local. Physical and queue convergence - // above establish recovery; a lost task must not report success. - assert_eq!( - task_status["summary"].as_str(), - Some("notFound"), - "interrupted task status: {task_status}" - ); - assert_eq!( - task_status["detail"].as_str(), - Some("heal task not found or expired"), - "interrupted admin task must be explicitly unavailable: {task_status}" - ); + // Restart recovery must finish the original durable root request. info!( event = "heal_interruption_recovered", component = "e2e_test", subsystem = "heal", interruption_node, interruption_kind, - task_state = "not_found", - "Physical recovery completed after coordinator restart" + task_state = "finished", + "Original root heal completed after coordinator restart" ); return Ok(()); } - if task_status["summary"].as_str() != Some("finished") { - return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into()); - } if let Some(evidence_context) = evidence_run { let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id(); @@ -1689,17 +2124,46 @@ mod tests { evidence_context.run.binary.sha256, "server build changed during restart" ); - let evidence = serde_json::json!({ + let outage_write_diagnostic = (!outage_target_manifest_required).then(|| { + serde_json::json!({ + "attempted": true, + "required": false, + "accepted": true, + "attempts": if outage_write_deferred_until_rejoin { + max_outage_write_attempts + } else { + service_unavailable_outage_writes + 1 + }, + "service_unavailable": service_unavailable_outage_writes, + "deferred_until_rejoin": outage_write_deferred_until_rejoin, + }) + }); + let mut evidence = serde_json::json!({ "schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence, "run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision, "test_build": compiled_test_identity(), "binary_sha256": evidence_context.run.binary.sha256, "test_binary_sha256": evidence_context.run.test_binary.sha256, "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()}, + "erasure_set_drive_count": erasure_set_drive_count, + "sets": topology.set_count(evidence_context.case.erasure_set_drive_count), + "pools": topology.pool_count(), + "rebuild_owner": if background_rejoin_heal_evidence { "background-erasure-set" } else { "admin-root-heal" }, + "outage_target_manifest_required": outage_target_manifest_required, + "outage_write_deferred_until_rejoin": outage_write_deferred_until_rejoin, + "admin_root_heal_takeover": !background_rejoin_heal_evidence, + "recovery_admin_task_restarted": recovery_admin_task_restarted, + "distributed_ec_invalidation": true, + "peer_count": cluster.nodes.len(), + "same_window_remote_proof": true, + "all_peers_bound_to_generation_window": true, "pid_before": target_pid, "pid_after": restarted_pid, "unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false), "objects": evidence_objects, "node_listings": node_listings, }); + if let Some(outage_write) = outage_write_diagnostic { + evidence["outage_write"] = outage_write; + } let data = serde_json::to_vec(&evidence)?; if data.len() > 1024 * 1024 { return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into()); diff --git a/crates/e2e_test/src/kms/kms_fault_recovery_test.rs b/crates/e2e_test/src/kms/kms_fault_recovery_test.rs index 2e66c9bf8..4fed36609 100644 --- a/crates/e2e_test/src/kms/kms_fault_recovery_test.rs +++ b/crates/e2e_test/src/kms/kms_fault_recovery_test.rs @@ -109,10 +109,10 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box Result<(), B async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box> { init_logging(); - let mut env = RustFSTestEnvironment::new().await?; - let master_key = local_sse_master_key_value(); - env.start_rustfs_server_with_env(vec![], &[(LOCAL_SSE_MASTER_KEY_ENV, master_key.as_str())]) - .await?; + let mut kms_env = LocalKMSTestEnvironment::new().await?; + let default_key_id = kms_env.start_rustfs_for_local_kms().await?; + kms_env.wait_for_kms_ready().await?; + let env = &kms_env.base_env; let bucket = "anon-post-default-sse-kms"; let object_key = "post-default-sse-kms-object.txt"; @@ -1484,7 +1485,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), .apply_server_side_encryption_by_default( ServerSideEncryptionByDefault::builder() .sse_algorithm(ServerSideEncryption::AwsKms) - .kms_master_key_id("test-key") + .kms_master_key_id(default_key_id) .build() .expect("default encryption rule should build"), ) diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index d6e7f0f0e..c9c196f23 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -17,6 +17,7 @@ use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use bytes::Bytes; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::sync::Barrier; use tracing::{info, warn}; @@ -25,6 +26,123 @@ const KEY: &str = "thumb/79/concurrent-overwrite.jpg"; type TestResult = Result<(), Box>; +async fn assert_degraded_cluster_publication_guard_errors_are_retryable() -> TestResult { + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2"); + cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); + cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false"); + cluster.set_env("RUST_LOG", "warn"); + cluster.start().await?; + cluster.create_test_bucket(BUCKET).await?; + let clients: Vec<_> = cluster + .create_all_clients()? + .into_iter() + .map(|client| { + Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ) + }) + .collect(); + + for alive in (1..=4).rev() { + if alive < 4 { + cluster.stop_node(alive)?; + } + for (node, client) in clients.iter().take(alive).enumerate() { + let key = format!("publication-put-{alive}-{node}"); + let put = client + .put_object() + .bucket(BUCKET) + .key(&key) + .body(Bytes::from_static(b"publication guard regression").into()) + .send() + .await; + if alive >= 3 { + put?; + } else { + let err = put.expect_err("PUT must reject writes without a write quorum"); + assert_eq!( + err.raw_response().map(|response| response.status().as_u16()), + Some(503), + "PUT with {alive} nodes alive, requested through node {node}: {err:?}" + ); + assert_eq!( + err.as_service_error().and_then(|error| error.meta().code()), + Some("ServiceUnavailable"), + "PUT with {alive} nodes alive, requested through node {node}: {err:?}" + ); + } + + let multipart_key = format!("publication-multipart-{alive}-{node}"); + let multipart = client + .create_multipart_upload() + .bucket(BUCKET) + .key(&multipart_key) + .send() + .await; + if alive >= 3 { + let upload = multipart?; + let upload_id = upload + .upload_id() + .expect("successful multipart initialization must return an upload ID"); + client + .abort_multipart_upload() + .bucket(BUCKET) + .key(&multipart_key) + .upload_id(upload_id) + .send() + .await?; + } else { + let err = multipart.expect_err("multipart initialization must reject writes without a write quorum"); + assert_eq!( + err.raw_response().map(|response| response.status().as_u16()), + Some(503), + "CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}" + ); + assert_eq!( + err.as_service_error().and_then(|error| error.meta().code()), + Some("ServiceUnavailable"), + "CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}" + ); + } + } + } + + cluster.stop(); + cluster.start().await?; + for client in &clients { + for alive in [1, 3, 4] { + for node in 0..alive { + let key = format!("publication-put-{alive}-{node}"); + let get = client.get_object().bucket(BUCKET).key(key).send().await; + if alive >= 3 { + assert_eq!( + get?.body.collect().await?.into_bytes().as_ref(), + b"publication guard regression", + "acknowledged writes must survive restart" + ); + } else { + let err = get.expect_err("a rejected publication guard must not publish an object"); + assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey")); + } + } + } + let uploads = client.list_multipart_uploads().bucket(BUCKET).send().await?; + assert!( + uploads + .uploads() + .iter() + .all(|upload| upload.key() != Some("publication-multipart-1-0")), + "a rejected publication guard must not publish a multipart upload" + ); + } + Ok(()) +} + async fn assert_quorum_object_body(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult { let body = client .get_object() @@ -242,6 +360,9 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() // Keep the regression focused on false quorum-loss errors, not ordinary lock // wait exhaustion under a heavily contended same-key overwrite workload. cluster.set_env("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "20"); + cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2"); + cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false"); + cluster.set_env("RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE", "false"); cluster.start().await?; cluster.create_test_bucket(BUCKET).await?; @@ -300,6 +421,181 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() ); clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?; + assert_node_readiness_tracks_quorum(&mut cluster).await?; + Ok(()) +} + +async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult { + let clients: Vec<_> = cluster + .create_all_clients()? + .into_iter() + .map(|client| { + Client::from_conf( + client + .config() + .to_builder() + .retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1)) + .build(), + ) + }) + .collect(); + let http = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(3)) + .build()?; + let seed_key = "readiness-seed"; + let seed_body = b"readiness quorum regression"; + clients[0] + .put_object() + .bucket(BUCKET) + .key(seed_key) + .body(Bytes::from_static(seed_body).into()) + .send() + .await?; + + for (phase, survivors) in [4, 3, 2, 1, 4].into_iter().enumerate() { + if phase == 4 { + cluster.stop(); + cluster.start().await?; + } else if survivors < 4 { + cluster.stop_node(survivors)?; + } + let write_ready = survivors >= 3; + let read_quorum = survivors >= 2; + let expected_status = if write_ready { 200 } else { 503 }; + for (idx, client) in clients.iter().enumerate().take(survivors) { + let url = &cluster.nodes[idx].url; + let deadline = Instant::now() + Duration::from_secs(30); + // Poll health before issuing S3 I/O: idle remote disk handles must + // not remain evidence of quorum after their host becomes unreachable. + let payload = loop { + let response = http.get(format!("{url}/health/ready")).send().await?; + let status = response.status().as_u16(); + let payload: serde_json::Value = response.json().await?; + if status == expected_status + && payload["ready"] == write_ready + && payload["details"]["storage"]["ready"] == write_ready + && payload["details"]["storage"]["readQuorum"] == read_quorum + && payload["details"]["storage"]["writeQuorum"] == write_ready + && payload["details"]["poolMetadata"]["ready"] == true + && payload["details"]["iam"]["ready"] == true + && payload["details"]["lock"]["ready"] == write_ready + { + break payload; + } + assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}"); + tokio::time::sleep(Duration::from_millis(200)).await; + }; + assert_eq!(payload["details"]["storage"]["readinessScope"], "write_quorum_and_pool_metadata"); + assert_eq!(payload["details"]["storage"]["source"], "local_runtime"); + assert_eq!( + payload["details"]["storage"]["status"], + if write_ready { "connected" } else { "disconnected" } + ); + if !write_ready { + assert!( + payload["degradedReasons"] + .as_array() + .expect("degraded reasons") + .iter() + .any(|reason| reason == "storage_and_lock_unavailable") + ); + } + + for path in ["/health/ready", "/minio/health/ready"] { + let head = http.head(format!("{url}{path}")).send().await?; + assert_eq!(head.status().as_u16(), expected_status, "HEAD {path}, survivors={survivors}"); + assert!(head.bytes().await?.is_empty()); + let response = http.get(format!("{url}{path}")).send().await?; + assert_eq!(response.status().as_u16(), expected_status); + let body: serde_json::Value = response.json().await?; + assert_eq!(body["details"]["storage"], payload["details"]["storage"]); + assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]); + } + let live = http.get(format!("{url}/health/live")).send().await?; + assert_eq!(live.status().as_u16(), 200); + assert!(live.json::().await?.get("details").is_none()); + for (path, storage_ready, scope) in [ + ("/minio/health/cluster", write_ready, "write_quorum_and_pool_metadata"), + ("/minio/health/cluster/read", read_quorum, "read_quorum"), + ] { + let deadline = Instant::now() + Duration::from_secs(30); + // Cluster read/write reports have independent caches; allow + // each observation to expire before comparing stable states. + let body = loop { + let response = http.get(format!("{url}{path}")).send().await?; + let status = response.status().as_u16(); + let body: serde_json::Value = response.json().await?; + if status == expected_status + && body["details"]["storage"]["ready"] == storage_ready + && body["details"]["lock"]["ready"] == write_ready + { + break body; + } + assert!(Instant::now() < deadline, "{path}, survivors={survivors}: HTTP {status}, {body}"); + tokio::time::sleep(Duration::from_millis(200)).await; + }; + assert_eq!(body["details"]["storage"]["readinessScope"], scope); + } + + let put = client + .put_object() + .bucket(BUCKET) + .key(format!("readiness-phase-{phase}-node-{idx}")) + .body(Bytes::from_static(seed_body).into()) + .send() + .await; + let put_status = if write_ready { + put.expect("a ready node must accept the PUT"); + 200 + } else { + let error = put.expect_err("subquorum node must reject PUT"); + assert!( + error.raw_response().is_some_and(|response| response.status().as_u16() >= 500), + "unexpected PUT failure: {error:?}" + ); + error.raw_response().expect("PUT error response").status().as_u16() + }; + let get = client.get_object().bucket(BUCKET).key(seed_key).send().await; + let get_status = match get { + Ok(object) => { + assert_eq!(object.body.collect().await?.into_bytes().as_ref(), seed_body); + 200 + } + Err(error) => { + assert!(!write_ready, "GET must succeed on a ready cluster: {error:?}"); + let status = error + .raw_response() + .expect("GET should have an HTTP response") + .status() + .as_u16(); + assert!(status >= 500, "unexpected GET failure: {error:?}"); + status + } + }; + let list = client.list_objects_v2().bucket(BUCKET).send().await; + let list_status = match list { + Ok(result) => { + assert!(result.contents().iter().any(|object| object.key() == Some(seed_key))); + 200 + } + Err(error) => { + assert!(!write_ready, "listing must succeed on a ready cluster: {error:?}"); + let status = error + .raw_response() + .expect("LIST should have an HTTP response") + .status() + .as_u16(); + assert!(status >= 500, "unexpected listing failure: {error:?}"); + status + } + }; + eprintln!( + "readiness matrix: survivors={survivors}, node={idx}, ready={write_ready}, read_quorum={read_quorum}, PUT={put_status}, GET={get_status}, LIST={list_status}" + ); + } + } + cluster.stop(); Ok(()) } @@ -309,6 +605,8 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() /// Before the fix, `map_namespace_lock_error` wrapped lock timeout/conflict errors as /// `StorageError::other(...)` → `StorageError::Io(...)`, which fell through to /// `S3ErrorCode::InternalError` (500) in the error mapping. +/// Also checks PUT and multipart initialization when node failures prevent +/// acquiring a table publication guard. #[tokio::test] async fn test_concurrent_put_same_key_never_returns_500() -> TestResult { crate::common::init_logging(); @@ -411,5 +709,6 @@ async fn test_concurrent_put_same_key_never_returns_500() -> TestResult { ); clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?; - Ok(()) + cluster.stop(); + assert_degraded_cluster_publication_guard_errors_are_retryable().await } diff --git a/crates/e2e_test/src/storage_metric_ownership_test.rs b/crates/e2e_test/src/storage_metric_ownership_test.rs new file mode 100644 index 000000000..68f46fae6 --- /dev/null +++ b/crates/e2e_test/src/storage_metric_ownership_test.rs @@ -0,0 +1,335 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Native OTLP -> Collector -> Prometheus contract, including a rolling upgrade. + +use crate::common::RustFSTestClusterEnvironment; +use aws_sdk_s3::primitives::ByteStream; +use serde_json::Value; +use std::fs::{self, File}; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +type TestResult = Result>; + +struct ToolProcess(Child); +impl Drop for ToolProcess { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn required_binary(name: &str) -> TestResult { + let path = PathBuf::from(std::env::var(name).map_err(|_| format!("{name} must name a pinned executable"))?); + if !path.is_file() { + return Err(format!("{name} does not name a file: {}", path.display()).into()); + } + Ok(path) +} + +fn free_port() -> TestResult { + Ok(TcpListener::bind("127.0.0.1:0")?.local_addr()?.port()) +} + +fn start_tool(binary: &Path, args: &[String], log: &Path) -> TestResult { + let log = File::create(log)?; + Ok(ToolProcess( + Command::new(binary) + .args(args) + .env("NO_PROXY", "127.0.0.1,localhost") + .env_remove("HTTP_PROXY") + .env_remove("HTTPS_PROXY") + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)) + .spawn()?, + )) +} + +async fn query(client: &reqwest::Client, base: &str, expression: &str) -> TestResult { + let mut url = reqwest::Url::parse(&format!("{base}/api/v1/query"))?; + url.query_pairs_mut().append_pair("query", expression); + let result: Value = client.get(url).send().await?.error_for_status()?.json().await?; + if result["status"] != "success" { + return Err(format!("PromQL failed: {result}").into()); + } + Ok(result["data"]["result"].clone()) +} + +async fn await_count(client: &reqwest::Client, base: &str, selector: &str, expected: u64) -> TestResult { + let deadline = Instant::now() + Duration::from_secs(120); + loop { + let result = query(client, base, &format!("count({selector}) or vector(0)")).await; + if let Ok(rows) = &result + && rows[0]["value"][1].as_str().and_then(|value| value.parse::().ok()) == Some(expected) + { + println!("PASS count={expected}: {selector}"); + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!("expected {expected} for {selector}; last result: {result:?}").into()); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +async fn validate_dashboard_queries(client: &reqwest::Client, base: &str, observer: &str) -> TestResult { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.docker/observability/grafana/dashboards/rustfs.json"); + let dashboard: Value = serde_json::from_str(&fs::read_to_string(path)?)?; + for name in ["storage_cluster", "storage_observer"] { + let variable = dashboard["templating"]["list"] + .as_array() + .ok_or("dashboard variables")? + .iter() + .find(|variable| variable["name"] == name) + .ok_or("storage selection variable")?; + assert_eq!(variable["multi"], false, "storage views must select one {name}"); + assert_eq!(variable["includeAll"], false, "storage views must select one {name}"); + } + let mut pending = dashboard["panels"] + .as_array() + .ok_or("dashboard panels")? + .iter() + .collect::>(); + let mut checked = 0; + while let Some(panel) = pending.pop() { + if let Some(children) = panel["panels"].as_array() { + pending.extend(children); + } + for target in panel["targets"].as_array().into_iter().flatten() { + let Some(expression) = target["expr"].as_str() else { continue }; + if !expression.contains("rustfs:storage:current") { + continue; + } + if expression.contains("collection_scope=\"cluster\"") { + assert!( + expression.contains("observer=\"$storage_observer\""), + "global views must select one observer: {expression}" + ); + } + let mut expression = expression.to_string(); + for (name, value) in [ + ("$__rate_interval", "5m"), + ("$storage_cluster", "metrics-e2e"), + ("$storage_observer", observer), + ("$drive_api", ".*"), + ("$server", ".*"), + ("$drive", ".*"), + ("$job", "rustfs"), + ] { + expression = expression.replace(name, value); + } + query(client, base, &expression).await?; + checked += 1; + } + } + assert!(checked > 0, "the storage dashboard queries must be exercised"); + println!("PASS: {checked} storage dashboard queries against the live pipeline"); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "external tools: pinned Collector, Prometheus, previous release and current RustFS binaries"] +async fn storage_metric_ownership_pipeline() -> TestResult { + let baseline = required_binary("RUSTFS_METRICS_BASELINE_BINARY")?; + let current = required_binary("CARGO_BIN_EXE_rustfs")?; + let collector = required_binary("RUSTFS_OTELCOL_BINARY")?; + let prometheus = required_binary("RUSTFS_PROMETHEUS_BINARY")?; + let temp = tempfile::Builder::new().prefix("rustfs-storage-metrics-").tempdir()?; + let work = if let Ok(path) = std::env::var("RUSTFS_METRICS_E2E_ARTIFACTS") { + let path = PathBuf::from(path); + fs::create_dir_all(&path)?; + tempfile::Builder::new().prefix("storage-run-").tempdir_in(path)?.keep() + } else { + temp.path().to_path_buf() + }; + println!("Metrics pipeline logs: {}", work.display()); + let otlp = free_port()?; + let scrape = free_port()?; + let prom = free_port()?; + let collector_config = work.join("collector.yaml"); + fs::write( + &collector_config, + format!( + r#"receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:{otlp} +exporters: + prometheus: + endpoint: 127.0.0.1:{scrape} + send_timestamps: true + metric_expiration: 5m + resource_to_telemetry_conversion: + enabled: true +service: + telemetry: + metrics: + level: none + logs: + level: warn + pipelines: + metrics: + receivers: [otlp] + exporters: [prometheus] +"# + ), + )?; + let rules = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../.docker/observability/prometheus-rules/rustfs-storage.yml") + .canonicalize()?; + let prom_config = work.join("prometheus.yaml"); + fs::write( + &prom_config, + format!( + r#"global: + scrape_interval: 1s + evaluation_interval: 1s +rule_files: + - '{}' +scrape_configs: + - job_name: rustfs + static_configs: + - targets: ['127.0.0.1:{scrape}'] +"#, + rules.display() + ), + )?; + // Use the shipped expressions, with only the test evaluation interval shortened. + let test_rules = work.join("storage-rules.yaml"); + fs::write(&test_rules, fs::read_to_string(&rules)?.replace("interval: 15s", "interval: 1s"))?; + fs::write( + &prom_config, + fs::read_to_string(&prom_config)?.replace(&rules.display().to_string(), &test_rules.display().to_string()), + )?; + let _collector = start_tool( + &collector, + &[format!("--config={}", collector_config.display())], + &work.join("collector.log"), + )?; + let _prometheus = start_tool( + &prometheus, + &[ + format!("--config.file={}", prom_config.display()), + format!("--web.listen-address=127.0.0.1:{prom}"), + format!("--storage.tsdb.path={}", work.join("prometheus-data").display()), + ], + &work.join("prometheus.log"), + )?; + let http = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build()?; + let prom_url = format!("http://127.0.0.1:{prom}"); + await_count(&http, &prom_url, "up{job=\"rustfs\"} == 1", 1).await?; + + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + cluster.set_env("NO_PROXY", "127.0.0.1,localhost"); + cluster.set_env("RUSTFS_OBS_METRIC_ENDPOINT", format!("http://127.0.0.1:{otlp}/v1/metrics")); + cluster.set_env("OTEL_RESOURCE_ATTRIBUTES", "rustfs.cluster.id=metrics-e2e"); + cluster.set_env("RUSTFS_OBS_METER_INTERVAL", "2"); + cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "true"); + cluster.set_env("RUSTFS_OBS_LOGS_EXPORT_ENABLED", "false"); + cluster.set_env("RUSTFS_OBS_TRACES_EXPORT_ENABLED", "false"); + cluster.set_env("RUSTFS_METRICS_NODE_INTERVAL", "2"); + cluster.set_env("RUSTFS_METRICS_CLUSTER_INTERVAL", "5"); + for index in 0..4 { + // Four localhost processes otherwise share the startup resource IP. + // Distinct test host IDs model the four hosts in a distributed deployment. + cluster.set_node_env( + index, + "OTEL_RESOURCE_ATTRIBUTES", + format!("rustfs.cluster.id=metrics-e2e,host.id=metrics-node-{index}"), + )?; + cluster.set_node_capture_log_path(index, work.join(format!("node-{index}.log")).display().to_string())?; + } + cluster.start_with_binary(&baseline).await?; + // Reproduce the original four observers x four global drives before fixing it. + await_count(&http, &prom_url, "rustfs_system_drive_total_bytes{collection_scope=\"\",drive!=\"\"}", 16).await?; + let local = "rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"metrics-e2e\"}"; + for index in 0..4 { + cluster.stop_node_gracefully(index).await?; + cluster.start_node_from_binary(index, ¤t).await?; + await_count(&http, &prom_url, local, u64::try_from(index + 1)?).await?; + } + let rows = query(&http, &prom_url, local).await?; + for row in rows.as_array().ok_or("expected a metric vector")? { + assert_eq!( + row["metric"]["observer"], row["metric"]["server"], + "a node must only export its own detailed drives" + ); + } + let observer = cluster.nodes[0].address.clone(); + let inventory = format!( + "rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_present\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\"}}" + ); + await_count(&http, &prom_url, &inventory, 4).await?; + cluster.create_test_bucket("metrics-ownership").await?; + let client = cluster.create_s3_client(0)?; + for index in 0..8 { + client + .put_object() + .bucket("metrics-ownership") + .key(format!("object-{index}")) + .body(ByteStream::from(vec![7_u8; 4096])) + .send() + .await?; + } + await_count( + &http, + &prom_url, + "count by (server) (rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"})", + 4, + ).await?; + let counters = query( + &http, + &prom_url, + "rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"}", + ) + .await?; + assert!( + !counters.as_array().ok_or("expected counters")?.is_empty(), + "exercise actual storage counters" + ); + for row in counters.as_array().ok_or("expected counters")? { + assert_eq!(row["metric"]["observer"], row["metric"]["server"]); + assert!( + !row["metric"]["disk_id"].as_str().unwrap_or_default().is_empty(), + "counters must carry physical disk identity" + ); + } + validate_dashboard_queries(&http, &prom_url, &observer).await?; + for index in (1..4).rev() { + cluster.stop_node(index)?; + // The Collector stays alive; cached samples must not keep stopped owners fresh. + await_count(&http, &prom_url, local, u64::try_from(index)?).await?; + await_count(&http, &prom_url, &inventory, 4).await?; + let unavailable = format!( + "rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\",state=~\"offline|unknown|suspect\"}} == 1" + ); + await_count(&http, &prom_url, &unavailable, u64::try_from(4 - index)?).await?; + } + cluster.stop(); + await_count(&http, &prom_url, local, 0).await?; + cluster.start_with_binary(¤t).await?; + await_count(&http, &prom_url, local, 4).await?; + await_count(&http, &prom_url, &inventory, 4).await?; + let restored = client.get_object().bucket("metrics-ownership").key("object-0").send().await?; + assert_eq!(restored.body.collect().await?.into_bytes().as_ref(), vec![7_u8; 4096].as_slice()); + println!("PASS: baseline duplication; four rolling upgrades; owner identity; counters; 4 -> 3 -> 2 -> 1 -> 0 -> 4 recovery"); + Ok(()) +} diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs index 390b528ce..6b89f8148 100644 --- a/crates/e2e_test/src/upgrade_compatibility_test.rs +++ b/crates/e2e_test/src/upgrade_compatibility_test.rs @@ -32,8 +32,10 @@ use aws_sdk_s3::types::{ VersioningConfiguration, }; use http::{Method, StatusCode}; +use serde_json::Value; +use std::io::Write; use std::path::{Path, PathBuf}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::task::JoinSet; use tokio::time::{Instant, sleep}; @@ -41,6 +43,7 @@ type TestResult = Result<(), Box>; type BoxError = Box; const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY"; +const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR"; const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5"; const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI="; @@ -81,6 +84,14 @@ const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30); // of treating it as an upgrade failure. const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90); +struct G09EvidenceContext { + directory: PathBuf, + current_revision: String, + previous_revision: String, + run_id: String, + measurement_window_id: String, +} + fn source_binary() -> Result> { let path = std::env::var_os(SOURCE_BINARY_ENV) .map(PathBuf::from) @@ -91,6 +102,155 @@ fn source_binary() -> Result> Ok(path) } +fn is_lower_hex_revision(value: &str) -> bool { + value.len() == 40 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +fn current_source_revision() -> Result { + let revision = env!("RUSTFS_E2E_BUILD_COMMIT"); + if !is_lower_hex_revision(revision) { + return Err(format!("current test binary has invalid source revision {revision}").into()); + } + Ok(revision.to_string()) +} + +async fn binary_source_revision(binary: &Path) -> Result { + let output = tokio::process::Command::new(binary).arg("--version").output().await?; + if !output.status.success() { + return Err(format!("{} --version failed with {}", binary.display(), output.status).into()); + } + let version = String::from_utf8(output.stdout)?; + source_revision_from_version_output(&version, binary) +} + +fn source_revision_from_version_output(version: &str, binary: &Path) -> Result { + version + .split(|ch: char| !ch.is_ascii_hexdigit()) + .find(|token| is_lower_hex_revision(token)) + .map(str::to_string) + .ok_or_else(|| format!("{} --version did not expose a 40-byte source revision", binary.display()).into()) +} + +async fn g09_evidence_context(previous_binary: &Path) -> Result, BoxError> { + let Some(directory) = std::env::var_os(G09_EVIDENCE_DIR_ENV) else { + return Ok(None); + }; + let directory = PathBuf::from(directory); + std::fs::create_dir_all(&directory)?; + let current_revision = current_source_revision()?; + let previous_revision = binary_source_revision(previous_binary).await?; + if current_revision == previous_revision { + return Err("G09 mixed-version evidence requires distinct current and previous source revisions".into()); + } + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + Ok(Some(G09EvidenceContext { + directory, + current_revision, + previous_revision, + run_id: format!("g09-upgrade-{}-{now}", std::process::id()), + measurement_window_id: format!("g09-mixed-version-window-{now}"), + })) +} + +fn write_g09_evidence( + context: &G09EvidenceContext, + field: &str, + role: &str, + cases: &[&str], + test: &str, + details: Value, +) -> TestResult { + let path = context.directory.join(format!("G09-{field}.json")); + let mut evidence = serde_json::json!({ + "schema": 1, + "evidence_type": "measured", + "artifact_kind": "upgrade-compatibility-e2e", + "source_revision": context.current_revision, + "run_id": context.run_id, + "measurement_window_id": context.measurement_window_id, + "gate": "G09", + "field": field, + "versions": [context.previous_revision, context.current_revision], + "current_revision": context.current_revision, + "previous_revision": context.previous_revision, + "mixed_version_role": role, + "mixed_version_cases": cases, + "test": test, + "details": details, + }); + if field == "rollback_payload_evidence" { + evidence["rollback_payload_replayed"] = Value::Bool(true); + } + let data = serde_json::to_vec_pretty(&evidence)?; + if data.len() > 1024 * 1024 { + return Err("G09 mixed-version evidence exceeds the 1 MiB artifact budget".into()); + } + let mut output = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?; + output.write_all(&data)?; + output.write_all(b"\n")?; + output.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod g09_evidence_tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult { + let binary = Path::new("rustfs"); + assert_eq!( + source_revision_from_version_output("rustfs 1.0.0 abcdef0123456789abcdef0123456789abcdef01 clean", binary)?, + "abcdef0123456789abcdef0123456789abcdef01" + ); + assert!(source_revision_from_version_output("rustfs ABCDEF0123456789ABCDEF0123456789ABCDEF01", binary).is_err()); + assert!(source_revision_from_version_output("rustfs abcdef", binary).is_err()); + Ok(()) + } + + #[test] + fn g09_evidence_writer_records_measured_role_and_refuses_overwrite() -> TestResult { + let directory = TempDir::new()?; + let context = G09EvidenceContext { + directory: directory.path().to_path_buf(), + current_revision: "b".repeat(40), + previous_revision: "a".repeat(40), + run_id: "g09-upgrade-test-run".to_string(), + measurement_window_id: "g09-upgrade-test-window".to_string(), + }; + write_g09_evidence( + &context, + "mixed_version_reader_evidence", + "mixed-version-reader", + &["old-writer-new-reader", "new-writer-old-reader"], + "unit", + serde_json::json!({"assertions": ["reader evidence"]}), + )?; + let path = directory.path().join("G09-mixed_version_reader_evidence.json"); + let evidence: Value = serde_json::from_slice(&std::fs::read(path)?)?; + assert_eq!(evidence["evidence_type"], "measured"); + assert_eq!(evidence["gate"], "G09"); + assert_eq!(evidence["field"], "mixed_version_reader_evidence"); + assert_eq!(evidence["mixed_version_role"], "mixed-version-reader"); + assert_eq!(evidence["versions"], serde_json::json!(["a".repeat(40), "b".repeat(40)])); + + let overwrite = write_g09_evidence( + &context, + "mixed_version_reader_evidence", + "mixed-version-reader", + &["old-writer-new-reader", "new-writer-old-reader"], + "unit", + serde_json::json!({}), + ); + assert!(overwrite.is_err(), "G09 evidence must not overwrite an existing artifact"); + Ok(()) + } +} + async fn enable_versioning(client: &Client, bucket: &str) -> TestResult { let configuration = VersioningConfiguration::builder() .status(BucketVersioningStatus::Enabled) @@ -523,6 +683,7 @@ async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult { async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestResult { init_logging(); let previous_binary = source_binary()?; + let evidence_context = g09_evidence_context(&previous_binary).await?; let current_binary = rustfs_binary_path(); let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?; cluster.set_env("RUST_LOG", "rustfs=warn,rustfs_notify=warn"); @@ -555,6 +716,45 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes } } + if let Some(context) = evidence_context.as_ref() { + let phases = ["one-current-node", "one-previous-node"]; + let objects_per_phase = MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER + 2; + write_g09_evidence( + context, + "mixed_version_reader_evidence", + "mixed-version-reader", + &["old-writer-new-reader", "new-writer-old-reader"], + "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts", + serde_json::json!({ + "bucket": MIXED_BUCKET, + "phases": phases, + "objects_per_phase": objects_per_phase, + "assertions": [ + "current node reads objects written through previous-release client", + "previous-release node reads objects written through current client", + "all nodes list every mixed-version object after homogeneous-current convergence" + ], + }), + )?; + write_g09_evidence( + context, + "mixed_version_writer_evidence", + "mixed-version-writer", + &["old-reader-new-writer", "new-reader-old-writer"], + "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts", + serde_json::json!({ + "bucket": MIXED_BUCKET, + "phases": phases, + "objects_per_phase": objects_per_phase, + "assertions": [ + "current writer publishes objects readable by previous-release node", + "previous-release writer publishes objects readable by current node", + "multipart writers continue under one-current-node and one-previous-node layouts" + ], + }), + )?; + } + Ok(()) } @@ -1136,6 +1336,7 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() - async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult { init_logging(); let previous_binary = source_binary()?; + let evidence_context = g09_evidence_context(&previous_binary).await?; let replication_target = FakeS3Target::start().await?; replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET); @@ -1204,6 +1405,54 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes ); assert_eq!(body, post_rollback_bytes); + env.restart_server_preserving_data(vec![], &server_env).await?; + let current_again = env.create_s3_client(); + assert_versioning_enabled(¤t_again, ROLLBACK_BUCKET, "after rolling forward again").await?; + assert_default_sse_s3_encryption(¤t_again, ROLLBACK_BUCKET, "after rolling forward again").await?; + assert_bucket_tag(¤t_again, ROLLBACK_BUCKET, "after rolling forward again").await?; + assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after rolling forward again").await?; + assert_eq!( + read_object(¤t_again, ROLLBACK_BUCKET, single_key, Some(&single_version)) + .await? + .1, + single_bytes + ); + assert_eq!( + read_object(¤t_again, ROLLBACK_BUCKET, multipart_key, None).await?.1, + multipart_bytes + ); + assert_eq!( + read_object(¤t_again, ROLLBACK_BUCKET, post_rollback_key, None).await?.1, + post_rollback_bytes + ); + + if let Some(context) = evidence_context.as_ref() { + write_g09_evidence( + context, + "rollback_payload_evidence", + "rollback-payload", + &["rollback-to-old", "rollback-to-new", "unknown-field-retained"], + "upgrade_compatibility_test::rollback_to_previous_release_reads_current_bucket_metadata", + serde_json::json!({ + "bucket": ROLLBACK_BUCKET, + "cases": { + "rollback-to-old": [ + "previous-release binary reads current-build versioning, SSE-S3, tags, replication target and objects", + "previous-release writer honors the decoded current-build encryption configuration" + ], + "rollback-to-new": [ + "current build reads the object written by the rolled-back previous release", + "current build reads the current-build single-part and multipart objects after rolling forward again" + ], + "unknown-field-retained": [ + "previous release skips current-build bucket metadata extension fields without dropping known bucket configuration", + "current build reads the retained bucket configuration after the previous-release round trip" + ] + }, + }), + )?; + } + replication_target.shutdown().await; Ok(()) } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 435b29225..65995b95a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -38,6 +38,16 @@ pub mod bucket { } pub mod lifecycle { + pub mod legacy_transition_state_reconcile { + pub use crate::bucket::lifecycle::legacy_transition_state_reconcile::{ + LegacyTransitionStateCopyRepresentation, LegacyTransitionStateMetadataAlias, LegacyTransitionStateReconcileError, + LegacyTransitionStateReconcileOutcome, LegacyTransitionStateReconcileReadiness, + LegacyTransitionStateReconcileRequest, LegacyTransitionStateReconcileResponse, + LegacyTransitionStateReconcileSelector, LegacyTransitionStateSetRepresentation, LegacyTransitionStateSource, + LegacyTransitionStateTarget, + }; + } + pub mod bucket_lifecycle_audit { pub use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; } @@ -260,18 +270,19 @@ pub mod bucket { ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, - ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, - VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent, - complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id, - get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication, + ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, ScannerDirtyUsageMutationObserver, + ScannerDirtyUsageMutationSource, TargetReplicationResyncStatus, VersionPurgeStatusType, XferStats, + assign_site_replication_rule_priorities, commit_force_delete_intent, complete_force_delete_intent, + delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool, + get_global_replication_stats, get_proxy_targets, init_background_replication, invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id, - should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info, - should_use_existing_delete_replication_source, site_replication_rule_deployment_id, - unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, - version_purge_status_to_filemeta, + set_scanner_dirty_usage_mutation_observer, should_remove_replication_target, should_schedule_delete_replication, + should_use_existing_delete_replication_info, should_use_existing_delete_replication_source, + site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure, + validate_replication_config_target_arns, version_purge_status_to_filemeta, }; } @@ -366,6 +377,14 @@ pub mod config { } pub mod data_usage { + #[cfg(feature = "test-util")] + pub use crate::data_movement::SourceCleanupDeleteBarrier; + #[cfg(feature = "test-util")] + pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault; + pub use crate::data_movement::scanner_backlog::{ + MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner, + ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner, + }; pub use crate::data_usage::{ DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage, init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache, @@ -563,7 +582,7 @@ pub mod store_list { } pub mod storage { - pub use crate::core::pools::HealLifecycleExpiryContext; + pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME}; pub use crate::store::HealWalkVersion; pub use crate::store::{ BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 0bbde6629..8b33231af 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -966,6 +966,11 @@ async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: if let Some(err) = first_error { return Err(err); } + runtime_sources::notify_scanner_dirty_usage_mutation( + &oi.bucket, + &oi.name, + runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration, + ); Ok(true) } @@ -1215,7 +1220,7 @@ impl ExpiryState { while state.tasks_tx.len() < n { let (tx, rx) = mpsc::channel(EXPIRY_WORKER_QUEUE_CAPACITY); - let api = api.clone(); + let api = Arc::downgrade(&api); let rx = Arc::new(tokio::sync::Mutex::new(rx)); let stats = Arc::clone(&state.stats); let recovery_notify = Arc::clone(&state.recovery_notify); @@ -1243,14 +1248,18 @@ impl ExpiryState { async fn worker( rx: &mut Receiver>, - api: Arc, + api: Weak, stats: Arc, recovery_notify: Arc, ) { - let cancel_token = api.ctx.background_cancel_token().unwrap_or_else(|| { + let Some(initial_api) = api.upgrade() else { + return; + }; + let cancel_token = initial_api.ctx.background_cancel_token().unwrap_or_else(|| { static FALLBACK: std::sync::OnceLock = std::sync::OnceLock::new(); FALLBACK.get_or_init(tokio_util::sync::CancellationToken::new).clone() }); + drop(initial_api); loop { select! { @@ -1279,6 +1288,9 @@ impl ExpiryState { let v = v.expect("received None after None check"); stats.decrement_pending_tasks(); let _active_task = ExpiryActiveTask::begin(Arc::clone(&stats)); + let Some(api) = api.upgrade() else { + return; + }; if v.as_any().is::() { let v = v.as_any().downcast_ref::().expect("ExpiryTask downcast failed"); //debug!("lifecycle expiry worker received task: {:?}", v.obj_info); @@ -4749,6 +4761,11 @@ async fn expire_transitioned_object_with_lock_lost_signal( // Drop any cached restored-copy body so it does not sit resident // until TTL after the copy is expired (ODC-26). crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await; + runtime_sources::notify_scanner_dirty_usage_mutation( + &oi.bucket, + &oi.name, + runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration, + ); //audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn); Ok(dobj) } @@ -4784,6 +4801,11 @@ async fn expire_transitioned_object_with_lock_lost_signal( // The transitioned version is gone; evict any cached body for this object // so it does not linger until TTL (ODC-26). crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await; + runtime_sources::notify_scanner_dirty_usage_mutation( + &oi.bucket, + &oi.name, + runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration, + ); //audit_log_lifecycle(oi, ILMExpiry, tags); @@ -5165,6 +5187,9 @@ pub async fn put_restore_opts( user_defined: meta, version_id: oi.version_id.map(|e| e.to_string()), mod_time: oi.mod_time, + // Restore writes stored (possibly encrypted) bytes, so the writer's + // computed MD5 is not the object's public plaintext ETag. + preserve_etag: oi.etag.clone(), //expires: oi.expires, ..Default::default() }) @@ -7741,8 +7766,9 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(2); let worker_stats = Arc::clone(&stats); let worker_notify = Arc::clone(&recovery_notify); + let worker_store = Arc::downgrade(&ecstore); let worker = tokio::spawn(async move { - ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await; + ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await; }); let oi = ObjectInfo { bucket: "bucket".to_string(), @@ -7843,8 +7869,9 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(2); let worker_stats = Arc::clone(&stats); let worker_notify = Arc::clone(&recovery_notify); + let worker_store = Arc::downgrade(&ecstore); let worker = tokio::spawn(async move { - ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await; + ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await; }); stats.increment_pending_tasks(); @@ -8013,7 +8040,7 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(2); let worker_stats = Arc::clone(&stats); let worker_notify = Arc::clone(&recovery_notify); - let worker_store = Arc::clone(&ecstore); + let worker_store = Arc::downgrade(&ecstore); let worker = tokio::spawn(async move { ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await; }); @@ -8100,7 +8127,7 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(2); let worker_stats = Arc::clone(&stats); let worker_notify = Arc::clone(&recovery_notify); - let worker_store = Arc::clone(&ecstore); + let worker_store = Arc::downgrade(&ecstore); let worker = tokio::spawn(async move { ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await; }); @@ -8207,7 +8234,7 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(2); let worker_stats = Arc::clone(&stats); let worker_notify = Arc::clone(&recovery_notify); - let worker_store = Arc::clone(&ecstore); + let worker_store = Arc::downgrade(&ecstore); let worker = tokio::spawn(async move { ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await; }); @@ -8261,8 +8288,9 @@ mod tests { let (tx, mut rx) = tokio::sync::mpsc::channel(2); let worker_stats = Arc::clone(&stats); let worker_notify = Arc::clone(&recovery_notify); + let worker_store = Arc::downgrade(&ecstore); let worker = tokio::spawn(async move { - ExpiryState::worker(&mut rx, ecstore, worker_stats, worker_notify).await; + ExpiryState::worker(&mut rx, worker_store, worker_stats, worker_notify).await; }); let oi = ObjectInfo { bucket: format!("missing-bucket-{}", Uuid::new_v4()), diff --git a/crates/ecstore/src/bucket/lifecycle/legacy_transition_state_reconcile.rs b/crates/ecstore/src/bucket/lifecycle/legacy_transition_state_reconcile.rs new file mode 100644 index 000000000..5fd78211f --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/legacy_transition_state_reconcile.rs @@ -0,0 +1,1492 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Exact, generation-conditional reconciliation of legacy transition metadata. +//! Repairs are monotonic and never use metadata rollback or remote deletion. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::bucket::utils::check_bucket_and_object_names; +use crate::disk::{DiskAPI as _, TransitionStateReconcileCondition, UpdateMetadataOpts}; +use crate::object_api::ObjectOptions; +use crate::services::notification_sys::{ + LegacyTransitionStateReconcileFleetProofToken, acquire_cross_pool_fence_fleet_proof, + acquire_legacy_transition_state_reconcile_fleet_proof, acquire_remote_version_state_fleet_proof, + cross_pool_fence_fleet_proof_matches, cross_pool_fence_topology_generation, + legacy_transition_state_reconcile_fleet_proof_current, legacy_transition_state_reconcile_fleet_proof_matches, + remote_version_state_fleet_proof_matches, +}; +use crate::services::tier::tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata}; +use crate::services::tier::warm_backend::LegacyTransitionStateProbe; +use crate::set_disk::read_legacy_transition_state_metadata_copies; +use crate::store::ECStore; + +use rustfs_filemeta::{FileInfo, FileMeta, TRANSITION_COMPLETE, TransitionVersionState}; +use rustfs_utils::http::metadata_compat::{ + SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, + strip_internal_prefix_preserving_case, +}; + +const LIVE_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateReconcileSelector { + pub bucket: String, + pub object: String, + pub version_id: String, +} + +impl LegacyTransitionStateReconcileSelector { + fn canonicalize(mut self) -> Result<(Self, Option), LegacyTransitionStateReconcileError> { + check_bucket_and_object_names(&self.bucket, &self.object) + .map_err(|err| LegacyTransitionStateReconcileError::InvalidSelector(err.to_string()))?; + if self.version_id.is_empty() { + return Err(LegacyTransitionStateReconcileError::InvalidSelector( + "versionId is required; use the literal null for an unversioned object".to_string(), + )); + } + if self.version_id == "null" { + return Ok((self, None)); + } + let version_id = Uuid::parse_str(&self.version_id).map_err(|_| { + LegacyTransitionStateReconcileError::InvalidSelector( + "versionId must be a non-nil UUID or the literal null".to_string(), + ) + })?; + if version_id.is_nil() { + return Err(LegacyTransitionStateReconcileError::InvalidSelector( + "a nil UUID is not a valid selector; use the literal null".to_string(), + )); + } + self.version_id = version_id.to_string(); + Ok((self, Some(version_id))) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateMetadataAlias { + pub key: String, + /// Lowercase hexadecimal preserves empty and non-UTF-8 values exactly. + pub value_hex: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateSource { + pub bucket_incarnation: String, + /// Absent when the fleet has not supplied a current topology proof. + pub topology_generation: Option, + pub bucket: String, + pub object: String, + pub version_id: String, + pub data_dir: String, + pub modification_time_unix_nanos: i128, + pub size: i64, + pub etag: String, + pub transition_status: String, + pub tier: String, + pub remote_object: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateCopyRepresentation { + pub disk_index: usize, + pub metadata_digest: String, + pub unchanged_metadata_digest: String, + pub state_aliases: Vec, + pub version_aliases: Vec, + pub destination_aliases: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateSetRepresentation { + pub pool_index: usize, + pub set_index: usize, + pub total_copies: usize, + pub available_copies: usize, + pub copies: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateTarget { + pub state: TransitionVersionState, + pub remote_version: Option, + pub destination_id: String, + pub tier_generation: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateReconcileRequest { + pub confirm: bool, + pub selector: LegacyTransitionStateReconcileSelector, + pub source: LegacyTransitionStateSource, + pub original_sets: Vec, + pub target: LegacyTransitionStateTarget, + pub reconciliation_digest: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum LegacyTransitionStateReconcileOutcome { + ReadyToMigrate, + Migrated, + RetainedAmbiguous, + Corrupt, + BackendUnavailable, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateReconcileReadiness { + pub fleet_ready: bool, + pub topology_ready: bool, + pub tier_generation_ready: bool, + pub metadata_quorum_ready: bool, + pub post_ready: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LegacyTransitionStateReconcileResponse { + pub outcome: LegacyTransitionStateReconcileOutcome, + pub reason_code: String, + pub reason: String, + pub retryable: bool, + pub changed: bool, + /// A failed RPC may have committed even when its response/readback is lost. + #[serde(default)] + pub changes_indeterminate: bool, + pub selector: LegacyTransitionStateReconcileSelector, + pub source: Option, + pub original_sets: Vec, + pub target: Option, + pub reconciliation_digest: Option, + pub readiness: LegacyTransitionStateReconcileReadiness, +} + +#[derive(Debug, thiserror::Error)] +pub enum LegacyTransitionStateReconcileError { + #[error("invalid legacy transition state selector: {0}")] + InvalidSelector(String), + #[error("invalid legacy transition state reconciliation request: {0}")] + InvalidRequest(String), + #[error("legacy transition state reconciliation expected tuple is stale: {0}")] + StaleExpectedTuple(String), + #[error("legacy transition state metadata is corrupt: {0}")] + Corrupt(String), + #[error("legacy transition state backend is unavailable: {0}")] + BackendUnavailable(String), + #[error("legacy transition state write fence is unavailable: {0}")] + WriteFenceUnavailable(String), +} + +struct InspectedCopy { + file_info: FileInfo, + representation: LegacyTransitionStateCopyRepresentation, +} + +/// Ownership accompanies a local publication into its blocking executor. +/// Remote disks reconstruct fleet/backend ownership from current local state; +/// serialized options never carry an authority supplied by another process. +pub(crate) struct TransitionStateReconcileAuthority { + // Declaration order releases physical locks, tier, bucket, then fleet. + objects: Vec, + tier: TierOperationLease, + bucket: Option, + fleet: LegacyTransitionStateReconcileFleetProofToken, +} + +impl std::fmt::Debug for TransitionStateReconcileAuthority { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TransitionStateReconcileAuthority") + .finish_non_exhaustive() + } +} + +impl TransitionStateReconcileAuthority { + pub(crate) fn is_current(&self) -> bool { + legacy_transition_state_reconcile_fleet_proof_current(&self.fleet) + && self.tier.is_current_generation() + && self.bucket.as_ref().is_none_or(|bucket| !bucket.is_lock_lost()) + && self.objects.iter().all(|guard| !guard.is_lock_lost()) + } + + pub(crate) async fn for_disk(condition: &TransitionStateReconcileCondition) -> crate::disk::error::Result> { + use crate::disk::error::Error as DiskError; + if let Some(authority) = &condition.authority { + if authority.is_current() { + return Ok(Arc::clone(authority)); + } + return Err(DiskError::OutdatedXLMeta); + } + let fleet = acquire_legacy_transition_state_reconcile_fleet_proof() + .await + .ok_or(DiskError::OutdatedXLMeta)?; + let topology = acquire_cross_pool_fence_fleet_proof().ok_or(DiskError::OutdatedXLMeta)?; + if cross_pool_fence_topology_generation(&topology) != condition.topology_generation { + return Err(DiskError::OutdatedXLMeta); + } + let tier = TierConfigMgr::acquire_operation_lease(&crate::runtime::sources::global_tier_config_mgr(), &condition.tier) + .await + .map_err(|_| DiskError::OutdatedXLMeta)?; + if rustfs_utils::crypto::hex(tier.backend_identity()) != condition.target.destination_id { + return Err(DiskError::OutdatedXLMeta); + } + Ok(Arc::new(Self { + fleet, + tier, + objects: Vec::new(), + bucket: None, + })) + } +} + +fn digest_hex(bytes: &[u8]) -> String { + rustfs_utils::crypto::hex(Sha256::digest(bytes)) +} + +fn metadata_aliases(metadata: &HashMap>, suffix: &str) -> Vec { + let mut aliases = metadata + .iter() + .filter(|(key, _)| internal_suffix_matches(key, suffix)) + .map(|(key, value)| LegacyTransitionStateMetadataAlias { + key: key.clone(), + value_hex: rustfs_utils::crypto::hex(value), + }) + .collect::>(); + aliases.sort_by(|left, right| left.key.cmp(&right.key)); + aliases +} + +fn internal_suffix_matches(key: &str, suffix: &str) -> bool { + strip_internal_prefix_preserving_case(key).is_some_and(|candidate| candidate.eq_ignore_ascii_case(suffix)) +} + +fn inspect_copy( + raw: &[u8], + disk_index: usize, + bucket: &str, + object: &str, + version_id: Option, +) -> Result, LegacyTransitionStateReconcileError> { + let metadata = FileMeta::load(raw) + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(format!("xl.meta decode failed: {err}")))?; + let (_, version) = match metadata.find_version(version_id) { + Ok(version) => version, + Err(rustfs_filemeta::Error::FileVersionNotFound) => return Ok(None), + Err(err) => return Err(LegacyTransitionStateReconcileError::Corrupt(err.to_string())), + }; + let object_metadata = version.object.as_ref().ok_or_else(|| { + LegacyTransitionStateReconcileError::Corrupt("the selected local version is not an object version".to_string()) + })?; + validate_remote_version_bytes(&object_metadata.meta_sys)?; + let file_info = version + .into_fileinfo(bucket, object, true) + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(err.to_string()))?; + file_info + .validate_for_metadata_read() + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(err.to_string()))?; + Ok(Some(InspectedCopy { + file_info, + representation: LegacyTransitionStateCopyRepresentation { + disk_index, + metadata_digest: digest_hex(raw), + unchanged_metadata_digest: digest_hex( + &metadata + .transition_reconcile_generation(version_id) + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(err.to_string()))?, + ), + state_aliases: metadata_aliases(&object_metadata.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE), + version_aliases: metadata_aliases(&object_metadata.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID), + destination_aliases: metadata_aliases(&object_metadata.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID), + }, + })) +} + +fn validate_remote_version_bytes(metadata: &HashMap>) -> Result<(), LegacyTransitionStateReconcileError> { + for (key, value) in metadata + .iter() + .filter(|(key, _)| internal_suffix_matches(key, SUFFIX_TRANSITIONED_VERSION_ID)) + { + if value.is_empty() { + continue; + } + if let Ok(version_id) = Uuid::from_slice(value) { + if !version_id.is_nil() { + continue; + } + return Err(LegacyTransitionStateReconcileError::Corrupt(format!( + "legacy remote version alias {key} contains a nil UUID" + ))); + } + let value = std::str::from_utf8(value).map_err(|_| { + LegacyTransitionStateReconcileError::Corrupt(format!( + "legacy remote version alias {key} is not valid UTF-8 or a raw UUID" + )) + })?; + if value.len() > 1024 + || value.chars().any(char::is_control) + || Uuid::parse_str(value).is_ok_and(|version_id| version_id.is_nil()) + { + return Err(LegacyTransitionStateReconcileError::Corrupt(format!( + "legacy remote version alias {key} contains an invalid identifier" + ))); + } + } + Ok(()) +} + +fn source_from_file_info( + selector: &LegacyTransitionStateReconcileSelector, + bucket_incarnation: Uuid, + file_info: &FileInfo, +) -> Result { + let data_dir = file_info.data_dir.ok_or_else(|| { + LegacyTransitionStateReconcileError::Corrupt("the selected transition source is missing its data directory".to_string()) + })?; + let modification_time = file_info.mod_time.ok_or_else(|| { + LegacyTransitionStateReconcileError::Corrupt( + "the selected transition source is missing its modification time".to_string(), + ) + })?; + let etag = crate::object_api::object_api_utils::get_raw_etag(&file_info.metadata); + if etag.is_empty() { + return Err(LegacyTransitionStateReconcileError::Corrupt( + "the selected transition source is missing its ETag".to_string(), + )); + } + Ok(LegacyTransitionStateSource { + bucket_incarnation: bucket_incarnation.to_string(), + topology_generation: None, + bucket: selector.bucket.clone(), + object: selector.object.clone(), + version_id: selector.version_id.clone(), + data_dir: data_dir.to_string(), + modification_time_unix_nanos: modification_time.unix_timestamp_nanos(), + size: file_info.size, + etag, + transition_status: file_info.transition_status.clone(), + tier: file_info.transition_tier.clone(), + remote_object: file_info.transitioned_objname.clone(), + }) +} + +fn response_digest( + source: &LegacyTransitionStateSource, + sets: &[LegacyTransitionStateSetRepresentation], + target: &LegacyTransitionStateTarget, +) -> Result { + let encoded = serde_json::to_vec(&(source, sets, target)) + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(err.to_string()))?; + Ok(digest_hex(&encoded)) +} + +impl ECStore { + pub async fn inspect_legacy_transition_state( + &self, + selector: LegacyTransitionStateReconcileSelector, + ) -> Result { + let (canonical_selector, _) = selector.clone().canonicalize()?; + // Keep the multi-pool inspection off the request stack and erase its + // concrete future so admin callers do not repeat its Send proof. + let inspection: futures::future::BoxFuture< + '_, + Result, + > = Box::pin(self.inspect_legacy_transition_state_inner(canonical_selector.clone(), false, None)); + match inspection.await { + Ok(response) => Ok(response), + Err(err) => Ok(error_response(canonical_selector, err)), + } + } + + async fn inspect_legacy_transition_state_inner( + &self, + selector: LegacyTransitionStateReconcileSelector, + write_locked: bool, + bound_lease: Option<&TierOperationLease>, + ) -> Result { + let (selector, local_version_id) = selector.canonicalize()?; + let remote_fleet_proof = acquire_remote_version_state_fleet_proof(); + let topology_proof = acquire_cross_pool_fence_fleet_proof(); + // Snapshot lock order: bucket lifecycle READ, then the fixed and + // physical object READ domains in pool/set order. Release these locks + // before acquiring the tier lease or waiting on remote I/O. + let bucket_guard = if write_locked { + None + } else { + Some( + self.acquire_bucket_lifecycle_read_lock(&selector.bucket) + .await + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?, + ) + }; + let bucket_incarnation = self + .bucket_incarnation_id_from_disk(&selector.bucket) + .await + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?; + let encoded_object = rustfs_utils::path::encode_dir_object(&selector.object); + let mut lock_options = ObjectOptions::default(); + let object_guards = if write_locked { + Vec::new() + } else { + self.acquire_all_physical_object_read_locks( + "legacy_transition_state_reconcile_inspect", + &selector.bucket, + &encoded_object, + &mut lock_options, + ) + .await + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))? + }; + let fleet_ready = acquire_legacy_transition_state_reconcile_fleet_proof().await.is_some(); + let mut topology_ready = topology_proof.is_some(); + let mut source = None; + let mut original_sets = Vec::new(); + let mut canonical_file_info: Option = None; + let mut observed_file_infos = Vec::new(); + let mut observed_state_alias_presence = Vec::new(); + + for set in self.all_set_disks() { + let raw_copies = read_legacy_transition_state_metadata_copies(&set, &selector.bucket, &selector.object) + .await + .map_err(|err| match err { + crate::disk::error::Error::FileCorrupt => { + LegacyTransitionStateReconcileError::Corrupt("an xl.meta copy is corrupt".to_string()) + } + _ => LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()), + })?; + let total_copies = raw_copies.len(); + let available_copies = raw_copies.iter().filter(|copy| copy.is_some()).count(); + let mut representations = Vec::new(); + let mut set_file_infos = Vec::new(); + for (disk_index, raw) in raw_copies.into_iter().enumerate() { + let Some(raw) = raw else { continue }; + let Some(inspected) = inspect_copy(&raw, disk_index, &selector.bucket, &selector.object, local_version_id)? + else { + continue; + }; + if let Some(expected) = &canonical_file_info { + if !matches_immutable_transition_source(expected, &inspected.file_info) { + return Err(LegacyTransitionStateReconcileError::Corrupt( + "authoritative copies disagree on the immutable transition source tuple".to_string(), + )); + } + } else { + source = Some(source_from_file_info(&selector, bucket_incarnation, &inspected.file_info)?); + canonical_file_info = Some(inspected.file_info.clone()); + } + observed_state_alias_presence.push(!inspected.representation.state_aliases.is_empty()); + observed_file_infos.push(inspected.file_info.clone()); + set_file_infos.push(inspected.file_info.clone()); + representations.push(inspected.representation); + } + if !representations.is_empty() { + if representations.len() != available_copies { + return Err(LegacyTransitionStateReconcileError::BackendUnavailable( + "the selected version is missing from an existing metadata copy".to_string(), + )); + } + let required = required_reconcile_copy_quorum(&set_file_infos, set.default_write_quorum()); + if representations.len() < required { + return Err(LegacyTransitionStateReconcileError::BackendUnavailable(format!( + "pool {} set {} has only {} selected-version copies; {required} are required", + set.pool_index, + set.set_index, + representations.len() + ))); + } + original_sets.push(LegacyTransitionStateSetRepresentation { + pool_index: set.pool_index, + set_index: set.set_index, + total_copies, + available_copies, + copies: representations, + }); + } + } + if object_guards.iter().any(|guard| guard.is_lock_lost()) + || bucket_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) + { + return Err(LegacyTransitionStateReconcileError::BackendUnavailable( + "a local metadata snapshot lock was lost before inspection completed".to_string(), + )); + } + drop(object_guards); + drop(bucket_guard); + + let Some(mut source) = source else { + return Err(LegacyTransitionStateReconcileError::Corrupt( + "the selected local object version was not found in any pool or set".to_string(), + )); + }; + source.topology_generation = topology_proof.as_ref().map(cross_pool_fence_topology_generation); + let file_info = canonical_file_info.ok_or_else(|| { + LegacyTransitionStateReconcileError::Corrupt( + "the selected source tuple disappeared while its metadata was being inspected".to_string(), + ) + })?; + if file_info.transition_status != TRANSITION_COMPLETE + || file_info.transition_tier.is_empty() + || file_info.transitioned_objname.is_empty() + { + return Ok(LegacyTransitionStateReconcileResponse { + outcome: LegacyTransitionStateReconcileOutcome::Corrupt, + reason_code: "partial_transition_tuple".to_string(), + reason: "the selected object does not contain a complete transition source tuple".to_string(), + retryable: false, + changed: false, + changes_indeterminate: false, + selector, + source: Some(source), + original_sets, + target: None, + reconciliation_digest: None, + readiness: readiness(fleet_ready, topology_ready, false, true, false), + }); + } + let has_explicit_unknown = + observed_file_infos + .iter() + .zip(&observed_state_alias_presence) + .any(|(observed, state_present)| { + observed.transition_version_state == TransitionVersionState::Unknown && *state_present + }); + if has_explicit_unknown { + return Ok(LegacyTransitionStateReconcileResponse { + outcome: LegacyTransitionStateReconcileOutcome::Corrupt, + reason_code: "explicit_unknown_state".to_string(), + reason: "an explicit unknown transition state is not legacy absence".to_string(), + retryable: false, + changed: false, + changes_indeterminate: false, + selector, + source: Some(source), + original_sets, + target: None, + reconciliation_digest: None, + readiness: readiness(fleet_ready, topology_ready, false, true, false), + }); + } + + let mut persisted_destination = None; + let mut persisted_remote_version: Option = None; + for observed in &observed_file_infos { + let destination = tier_destination_id_from_metadata(&observed.metadata) + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(err.to_string()))?; + if let Some(destination) = destination { + if persisted_destination.is_some_and(|expected| expected != destination) { + return Err(LegacyTransitionStateReconcileError::Corrupt( + "authoritative copies contain conflicting tier destination identities".to_string(), + )); + } + persisted_destination = Some(destination); + } + // A converged or partially migrated exact tuple still identifies + // its historical version, even when other remote versions exist. + if let Some(version) = observed.transition_version.as_deref() { + if persisted_remote_version + .as_deref() + .is_some_and(|expected| expected != version) + { + return Err(LegacyTransitionStateReconcileError::Corrupt( + "authoritative copies contain conflicting nonempty remote versions".to_string(), + )); + } + persisted_remote_version = Some(version.to_string()); + } + } + let tier_manager = self.tier_config_mgr(); + let owned_lease = if bound_lease.is_none() { + Some( + match persisted_destination { + Some(destination) => { + TierConfigMgr::acquire_operation_lease_for_backend_identity( + &tier_manager, + &file_info.transition_tier, + destination, + ) + .await + } + None => TierConfigMgr::acquire_operation_lease(&tier_manager, &file_info.transition_tier).await, + } + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?, + ) + } else { + None + }; + let lease = bound_lease.or(owned_lease.as_ref()).ok_or_else(|| { + LegacyTransitionStateReconcileError::BackendUnavailable("tier generation lease is unavailable".to_string()) + })?; + if persisted_destination.is_some_and(|identity| identity != lease.backend_identity()) { + return Err(LegacyTransitionStateReconcileError::Corrupt( + "persisted tier destination differs from the leased backend".to_string(), + )); + } + let tier_generation = lease.generation(); + let destination_id = rustfs_utils::crypto::hex(lease.backend_identity()); + let probe = tokio::time::timeout( + LIVE_PROBE_TIMEOUT, + lease.probe_legacy_transition_state(&file_info.transitioned_objname, persisted_remote_version.as_deref()), + ) + .await + .map_err(|_| LegacyTransitionStateReconcileError::BackendUnavailable("live tier probe timed out".to_string()))? + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?; + let remote_state_fleet_current = remote_fleet_proof + .as_ref() + .is_some_and(remote_version_state_fleet_proof_matches); + topology_ready = topology_proof.as_ref().is_some_and(cross_pool_fence_fleet_proof_matches) && remote_state_fleet_current; + if !lease.is_current_generation() + || self + .bucket_incarnation_id_from_disk(&selector.bucket) + .await + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))? + != bucket_incarnation + { + return Err(LegacyTransitionStateReconcileError::BackendUnavailable( + "the tier generation or bucket incarnation changed during the live probe".to_string(), + )); + } + let target = match probe { + LegacyTransitionStateProbe::UnversionedPresent => Some(LegacyTransitionStateTarget { + state: TransitionVersionState::KnownDisabled, + remote_version: None, + destination_id, + tier_generation, + }), + LegacyTransitionStateProbe::SuspendedNullPresent => Some(LegacyTransitionStateTarget { + state: TransitionVersionState::SuspendedNull, + remote_version: Some("null".to_string()), + destination_id, + tier_generation, + }), + LegacyTransitionStateProbe::VersionedPresent(version) => { + if version.is_empty() || version == "null" || Uuid::parse_str(&version).is_ok_and(|id| id.is_nil()) { + return Err(LegacyTransitionStateReconcileError::BackendUnavailable( + "the live tier probe returned an invalid exact version identifier".to_string(), + )); + } + rustfs_s3_client::provider_versions::validate_remote_version_id(&version) + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?; + lease + .validate_remote_version_id(&version) + .map_err(|err| LegacyTransitionStateReconcileError::Corrupt(err.to_string()))?; + Some(LegacyTransitionStateTarget { + state: TransitionVersionState::Exact, + remote_version: Some(version), + destination_id, + tier_generation, + }) + } + LegacyTransitionStateProbe::Missing + | LegacyTransitionStateProbe::Ambiguous + | LegacyTransitionStateProbe::Unsupported => None, + }; + let Some(target) = target else { + return Ok(LegacyTransitionStateReconcileResponse { + outcome: LegacyTransitionStateReconcileOutcome::RetainedAmbiguous, + reason_code: "live_probe_ambiguous".to_string(), + reason: "the live backend probe did not prove exactly one remote version model".to_string(), + retryable: true, + changed: false, + changes_indeterminate: false, + selector, + source: Some(source), + original_sets, + target: None, + reconciliation_digest: None, + readiness: readiness(fleet_ready, topology_ready, true, true, false), + }); + }; + + let already_explicit = file_info.transition_version_state == target.state + && observed_file_infos.iter().all(|observed| { + observed.transition_version_state == target.state + && observed.transition_version == target.remote_version + && tier_destination_id_from_metadata(&observed.metadata).ok().flatten() == Some(lease.backend_identity()) + }); + let legacy_remote_versions_match = observed_file_infos + .iter() + .zip(&observed_state_alias_presence) + .filter(|(observed, state_present)| { + observed.transition_version_state == TransitionVersionState::Unknown && !**state_present + }) + .all(|(observed, _)| legacy_remote_version_matches_target(observed, &target)); + if !legacy_remote_versions_match + || persisted_remote_version + .as_deref() + .is_some_and(|persisted| target.remote_version.as_deref() != Some(persisted)) + { + return Ok(LegacyTransitionStateReconcileResponse { + outcome: LegacyTransitionStateReconcileOutcome::RetainedAmbiguous, + reason_code: "legacy_remote_version_changed".to_string(), + reason: "the live backend candidate does not match the persisted nonempty legacy remote version".to_string(), + retryable: true, + changed: false, + changes_indeterminate: false, + selector, + source: Some(source), + original_sets, + target: None, + reconciliation_digest: None, + readiness: readiness(fleet_ready, topology_ready, true, true, false), + }); + } + let allowed_retry_subset = + observed_file_infos + .iter() + .zip(&observed_state_alias_presence) + .all(|(observed, state_present)| { + let destination = tier_destination_id_from_metadata(&observed.metadata).ok(); + let explicit_target = observed.transition_version_state == target.state + && observed.transition_version == target.remote_version + && matches!(destination, Some(Some(value)) if value == lease.backend_identity()); + let missing_destination_matches = match destination { + Some(None) => true, + Some(Some(value)) => value == lease.backend_identity(), + None => false, + }; + let missing_state = observed.transition_version_state == TransitionVersionState::Unknown + && !*state_present + && missing_destination_matches; + explicit_target || missing_state + }); + if !allowed_retry_subset { + return Ok(LegacyTransitionStateReconcileResponse { + outcome: LegacyTransitionStateReconcileOutcome::Corrupt, + reason_code: "conflicting_retry_subset".to_string(), + reason: "authoritative copies are outside the allowed original-missing plus exact-target retry subset" + .to_string(), + retryable: false, + changed: false, + changes_indeterminate: false, + selector, + source: Some(source), + original_sets, + target: Some(target), + reconciliation_digest: None, + readiness: readiness(fleet_ready, topology_ready, true, true, false), + }); + } + let digest = response_digest(&source, &original_sets, &target)?; + let post_ready = !already_explicit && fleet_ready && topology_ready; + Ok(LegacyTransitionStateReconcileResponse { + outcome: if already_explicit { + LegacyTransitionStateReconcileOutcome::Migrated + } else { + LegacyTransitionStateReconcileOutcome::ReadyToMigrate + }, + reason_code: if already_explicit { + "already_converged".to_string() + } else if post_ready { + "ready_to_migrate".to_string() + } else { + "fleet_write_capability_unavailable".to_string() + }, + reason: if already_explicit { + "all inspected copies already contain the proven explicit transition state".to_string() + } else if post_ready { + "the live backend probe proved a single target tuple".to_string() + } else { + "the target tuple is proven, but the required fleet write capability is not available".to_string() + }, + retryable: !already_explicit, + changed: false, + changes_indeterminate: false, + selector, + source: Some(source), + original_sets, + target: Some(target), + reconciliation_digest: Some(digest), + readiness: readiness(fleet_ready, topology_ready, true, true, post_ready), + }) + } + + pub async fn reconcile_legacy_transition_state( + &self, + request: LegacyTransitionStateReconcileRequest, + ) -> Result { + if !request.confirm { + return Err(LegacyTransitionStateReconcileError::InvalidRequest("confirm must be true".to_string())); + } + let (selector, _) = request.selector.clone().canonicalize()?; + if selector != request.selector { + return Err(LegacyTransitionStateReconcileError::InvalidRequest( + "selector UUID must use its canonical representation".to_string(), + )); + } + let expected_digest = response_digest(&request.source, &request.original_sets, &request.target)?; + if expected_digest != request.reconciliation_digest { + return Ok(LegacyTransitionStateReconcileResponse { + outcome: LegacyTransitionStateReconcileOutcome::Corrupt, + reason_code: "stale_expected_tuple".to_string(), + reason: "reconciliation digest does not match the supplied source, sets, and target".to_string(), + retryable: false, + changed: false, + changes_indeterminate: false, + selector: request.selector, + source: Some(request.source), + original_sets: request.original_sets, + target: Some(request.target), + reconciliation_digest: Some(request.reconciliation_digest), + readiness: unavailable_write_readiness(), + }); + } + let mut changed = false; + let mut indeterminate = false; + let result = Box::pin(self.apply_legacy_transition_state(&request, &mut changed, &mut indeterminate)).await; + let mut response = match result { + Ok(response) => response, + Err(err) => error_response(request.selector.clone(), err), + }; + response.changed = changed; + response.changes_indeterminate = indeterminate; + Ok(response) + } + + async fn apply_legacy_transition_state( + &self, + request: &LegacyTransitionStateReconcileRequest, + changed: &mut bool, + indeterminate: &mut bool, + ) -> Result { + let unavailable = |message: &str| LegacyTransitionStateReconcileError::WriteFenceUnavailable(message.to_string()); + let fleet = acquire_legacy_transition_state_reconcile_fleet_proof() + .await + .ok_or_else(|| unavailable("every metadata writer must support conditional transition reconciliation"))?; + // Admission -> bucket lifecycle WRITE -> exact tier generation -> + // all physical object WRITE domains -> disk metadata mutation domain. + let bucket = self + .acquire_bucket_lifecycle_write_lock(&request.selector.bucket) + .await + .map_err(|err| unavailable(&err.to_string()))?; + let tier = TierConfigMgr::acquire_operation_lease(&self.tier_config_mgr(), &request.source.tier) + .await + .map_err(|err| unavailable(&err.to_string()))?; + if tier.generation() != request.target.tier_generation + || rustfs_utils::crypto::hex(tier.backend_identity()) != request.target.destination_id + { + return Err(LegacyTransitionStateReconcileError::StaleExpectedTuple( + "tier generation or destination changed".to_string(), + )); + } + let object = rustfs_utils::path::encode_dir_object(&request.selector.object); + let objects = self + .acquire_all_physical_object_write_locks("legacy_transition_state_reconcile", &request.selector.bucket, &object) + .await + .map_err(|err| unavailable(&err.to_string()))?; + let authority = Arc::new(TransitionStateReconcileAuthority { + fleet, + tier, + objects, + bucket: Some(bucket), + }); + let mut current = self + .inspect_legacy_transition_state_inner(request.selector.clone(), true, Some(&authority.tier)) + .await?; + if !matches!( + current.outcome, + LegacyTransitionStateReconcileOutcome::ReadyToMigrate | LegacyTransitionStateReconcileOutcome::Migrated + ) { + return Ok(current); + } + validate_reconcile_snapshot(request, current.source.as_ref(), ¤t.original_sets, current.target.as_ref())?; + if !current.readiness.fleet_ready || !current.readiness.topology_ready { + return Err(unavailable("the current fleet/topology snapshot cannot authorize metadata writes")); + } + let topology = request + .source + .topology_generation + .as_ref() + .ok_or_else(|| unavailable("topology proof is absent"))?; + let (_, version_id) = request.selector.clone().canonicalize()?; + // Visit every observed copy, including an already committed retry + // subset. The second pass is a zero-write barrier in the same disk + // mutation domain; a delayed first attempt can only be idempotent. + for verify_only in [false, true] { + for set in self.all_set_disks() { + let Some(original_set) = request + .original_sets + .iter() + .find(|item| item.pool_index == set.pool_index && item.set_index == set.set_index) + else { + continue; + }; + let disks = set.disk_inventory().await; + for original in &original_set.copies { + if !legacy_transition_state_reconcile_fleet_proof_matches(&authority.fleet).await + || self + .bucket_incarnation_id_from_disk(&request.selector.bucket) + .await + .map_err(|err| unavailable(&err.to_string()))? + .to_string() + != request.source.bucket_incarnation + || !authority.is_current() + { + return Err(unavailable("fleet, tier, bucket, or object fence changed before metadata publication")); + } + let disk = disks + .get(original.disk_index) + .and_then(Option::as_ref) + .ok_or_else(|| unavailable("an authoritative disk is unavailable"))?; + let opts = UpdateMetadataOpts { + transition_reconcile: Some(Box::new(TransitionStateReconcileCondition { + expected_metadata_digest: original.metadata_digest.clone(), + unchanged_metadata_digest: original.unchanged_metadata_digest.clone(), + target: rustfs_filemeta::TransitionStateReconcileTarget { + state: request.target.state, + remote_version: request.target.remote_version.clone(), + destination_id: request.target.destination_id.clone(), + }, + tier: request.source.tier.clone(), + topology_generation: topology.clone(), + verify_only, + authority: Some(Arc::clone(&authority)), + })), + ..Default::default() + }; + let already_target = current + .original_sets + .iter() + .find(|item| item.pool_index == set.pool_index && item.set_index == set.set_index) + .and_then(|item| item.copies.iter().find(|copy| copy.disk_index == original.disk_index)) + .is_some_and(|copy| representation_matches_target(copy, &request.target)); + // Empty metadata makes a server which ignores the new + // conditional option reject the legacy update operation. + let result = disk + .update_metadata( + &request.selector.bucket, + &object, + FileInfo { + version_id, + ..Default::default() + }, + &opts, + ) + .await; + set.invalidate_get_object_metadata_cache(&request.selector.bucket, &object) + .await; + match result { + Ok(()) => { + *changed |= !verify_only && !already_target; + } + Err(err) => { + *indeterminate |= !verify_only && !already_target; + return Err(match err { + crate::disk::error::Error::OutdatedXLMeta => { + unavailable("a disk generation, rollback, or publication fence changed; inspect or retry") + } + crate::disk::error::Error::FileCorrupt => LegacyTransitionStateReconcileError::Corrupt( + "a disk rejected conflicting transition metadata".to_string(), + ), + _ => LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()), + }); + } + } + } + } + } + let (source, copies) = self.read_reconciled_transition_snapshot(&request.selector).await?; + validate_reconcile_snapshot(request, Some(&source), &copies, Some(&request.target))?; + if copies + .iter() + .flat_map(|set| &set.copies) + .any(|copy| !representation_matches_target(copy, &request.target)) + { + return Err(unavailable("strong readback did not prove convergence of every authoritative copy")); + } + if !legacy_transition_state_reconcile_fleet_proof_matches(&authority.fleet).await || !authority.is_current() { + return Err(unavailable("publication authority changed before final readback completed")); + } + current.outcome = LegacyTransitionStateReconcileOutcome::Migrated; + current.reason_code = if *changed { "migrated" } else { "already_converged" }.to_string(); + current.reason = "every authoritative metadata copy contains the proven state and destination".to_string(); + current.retryable = false; + current.reconciliation_digest = Some(response_digest(&source, &copies, &request.target)?); + current.source = Some(source); + current.original_sets = copies; + current.readiness = readiness(true, true, true, true, false); + Ok(current) + } + + async fn read_reconciled_transition_snapshot( + &self, + selector: &LegacyTransitionStateReconcileSelector, + ) -> Result<(LegacyTransitionStateSource, Vec), LegacyTransitionStateReconcileError> + { + let (_, version_id) = selector.clone().canonicalize()?; + let incarnation = self + .bucket_incarnation_id_from_disk(&selector.bucket) + .await + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?; + let topology = acquire_cross_pool_fence_fleet_proof() + .ok_or_else(|| LegacyTransitionStateReconcileError::WriteFenceUnavailable("topology proof expired".to_string()))?; + let mut source = None; + let mut sets = Vec::new(); + for set in self.all_set_disks() { + let raw = read_legacy_transition_state_metadata_copies(&set, &selector.bucket, &selector.object) + .await + .map_err(|err| LegacyTransitionStateReconcileError::BackendUnavailable(err.to_string()))?; + let total_copies = raw.len(); + let available_copies = raw.iter().flatten().count(); + let mut copies = Vec::new(); + for (index, raw) in raw.into_iter().enumerate() { + let Some(raw) = raw else { continue }; + let Some(copy) = inspect_copy(&raw, index, &selector.bucket, &selector.object, version_id)? else { + continue; + }; + let mut observed_source = source_from_file_info(selector, incarnation, ©.file_info)?; + observed_source.topology_generation = Some(cross_pool_fence_topology_generation(&topology)); + if source.as_ref().is_some_and(|source| source != &observed_source) { + return Err(LegacyTransitionStateReconcileError::StaleExpectedTuple( + "immutable source changed during readback".to_string(), + )); + } + source = Some(observed_source); + copies.push(copy.representation); + } + if !copies.is_empty() { + sets.push(LegacyTransitionStateSetRepresentation { + pool_index: set.pool_index, + set_index: set.set_index, + total_copies, + available_copies, + copies, + }); + } + } + Ok(( + source.ok_or_else(|| LegacyTransitionStateReconcileError::StaleExpectedTuple("source disappeared".to_string()))?, + sets, + )) + } +} + +fn representation_matches_target(copy: &LegacyTransitionStateCopyRepresentation, target: &LegacyTransitionStateTarget) -> bool { + let aliases_equal = |aliases: &[LegacyTransitionStateMetadataAlias], value: &str| { + !aliases.is_empty() + && aliases + .iter() + .all(|alias| alias.value_hex == rustfs_utils::crypto::hex(value.as_bytes())) + }; + aliases_equal(©.state_aliases, target.state.as_str()) + && aliases_equal(©.destination_aliases, &target.destination_id) + && match &target.remote_version { + Some(version) => aliases_equal(©.version_aliases, version), + None => copy.version_aliases.iter().all(|alias| alias.value_hex.is_empty()), + } +} + +fn validate_reconcile_snapshot( + request: &LegacyTransitionStateReconcileRequest, + source: Option<&LegacyTransitionStateSource>, + sets: &[LegacyTransitionStateSetRepresentation], + target: Option<&LegacyTransitionStateTarget>, +) -> Result<(), LegacyTransitionStateReconcileError> { + let matches = source == Some(&request.source) + && target == Some(&request.target) + && !sets.is_empty() + && sets.len() == request.original_sets.len() + && sets.iter().zip(&request.original_sets).all(|(current, original)| { + current.pool_index == original.pool_index + && current.set_index == original.set_index + && current.total_copies == original.total_copies + && current.available_copies == original.available_copies + && current.copies.len() == original.copies.len() + && current.copies.iter().zip(&original.copies).all(|(current, original)| { + current.disk_index == original.disk_index + && current.unchanged_metadata_digest == original.unchanged_metadata_digest + && (current == original || representation_matches_target(current, &request.target)) + }) + }); + if !matches { + return Err(LegacyTransitionStateReconcileError::StaleExpectedTuple( + "source, ownership, target, or unrelated metadata differs from the inspected generation".to_string(), + )); + } + Ok(()) +} + +fn legacy_remote_version_matches_target(file_info: &FileInfo, target: &LegacyTransitionStateTarget) -> bool { + match (&file_info.transition_version, &target.remote_version) { + (None, _) => true, + (Some(persisted), Some(proven)) => persisted == proven, + (Some(_), None) => false, + } +} + +fn unavailable_write_readiness() -> LegacyTransitionStateReconcileReadiness { + LegacyTransitionStateReconcileReadiness { + fleet_ready: false, + topology_ready: false, + tier_generation_ready: false, + metadata_quorum_ready: false, + post_ready: false, + } +} + +fn readiness( + fleet_ready: bool, + topology_ready: bool, + tier_generation_ready: bool, + metadata_quorum_ready: bool, + post_ready: bool, +) -> LegacyTransitionStateReconcileReadiness { + LegacyTransitionStateReconcileReadiness { + fleet_ready, + topology_ready, + tier_generation_ready, + metadata_quorum_ready, + post_ready, + } +} + +fn error_response( + selector: LegacyTransitionStateReconcileSelector, + err: LegacyTransitionStateReconcileError, +) -> LegacyTransitionStateReconcileResponse { + let (outcome, reason_code, retryable) = match &err { + LegacyTransitionStateReconcileError::Corrupt(_) | LegacyTransitionStateReconcileError::StaleExpectedTuple(_) => { + (LegacyTransitionStateReconcileOutcome::Corrupt, "corrupt", false) + } + LegacyTransitionStateReconcileError::BackendUnavailable(_) => { + (LegacyTransitionStateReconcileOutcome::BackendUnavailable, "backend_unavailable", true) + } + LegacyTransitionStateReconcileError::WriteFenceUnavailable(_) => { + (LegacyTransitionStateReconcileOutcome::BackendUnavailable, "write_fence_unavailable", true) + } + LegacyTransitionStateReconcileError::InvalidSelector(_) | LegacyTransitionStateReconcileError::InvalidRequest(_) => { + (LegacyTransitionStateReconcileOutcome::Corrupt, "invalid_request", false) + } + }; + LegacyTransitionStateReconcileResponse { + outcome, + reason_code: reason_code.to_string(), + reason: err.to_string(), + retryable, + changed: false, + changes_indeterminate: false, + selector, + source: None, + original_sets: Vec::new(), + target: None, + reconciliation_digest: None, + readiness: unavailable_write_readiness(), + } +} + +fn required_reconcile_copy_quorum(file_infos: &[FileInfo], default_write_quorum: usize) -> usize { + file_infos + .iter() + .map(|info| info.write_quorum(default_write_quorum)) + .max() + .unwrap_or(default_write_quorum) + .max(default_write_quorum) +} + +fn matches_immutable_transition_source(expected: &FileInfo, observed: &FileInfo) -> bool { + expected.version_id == observed.version_id + && expected.data_dir == observed.data_dir + && expected.mod_time == observed.mod_time + && expected.size == observed.size + && crate::object_api::object_api_utils::get_raw_etag(&expected.metadata) + == crate::object_api::object_api_utils::get_raw_etag(&observed.metadata) + && expected.transition_status == observed.transition_status + && expected.transition_tier == observed.transition_tier + && expected.transitioned_objname == observed.transitioned_objname +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pinned_minio_and_legacy_rustfs_envelopes_are_inspected_without_reencoding() { + for bytes in [ + rustfs_filemeta::test_data::create_minio_small_object_xlmeta().expect("pinned MinIO inline object"), + rustfs_filemeta::test_data::create_minio_large_object_xlmeta().expect("pinned MinIO external data object"), + rustfs_filemeta::test_data::create_minio_versioned_object_xlmeta().expect("pinned MinIO versioned object"), + rustfs_filemeta::test_data::create_issue_2265_legacy_meta_v2_object_xlmeta().expect("pinned legacy RustFS object"), + ] { + let original = bytes.clone(); + let metadata = FileMeta::load(&bytes).expect("load pinned metadata envelope"); + for version in &metadata.versions { + let inspected = inspect_copy(&bytes, 0, "bucket", "object", version.header.version_id); + if version.header.version_type == rustfs_filemeta::VersionType::Delete { + assert!( + matches!(inspected, Err(LegacyTransitionStateReconcileError::Corrupt(_))), + "a pinned delete marker is not a transition-state repair candidate" + ); + continue; + } + let inspected = inspected.expect("inspect pinned version").expect("pinned version is present"); + assert_eq!(inspected.file_info.transition_version_state, TransitionVersionState::Unknown); + assert!(inspected.representation.state_aliases.is_empty()); + assert_eq!(inspected.representation.metadata_digest, digest_hex(&original)); + } + assert_eq!(bytes, original, "foreign metadata must never be round-tripped by inspection"); + } + } + + fn legacy_metadata_fixture(version_bytes: Option<&[u8]>, state: Option<&[u8]>) -> Vec { + let mut metadata = FileMeta::new(); + metadata + .add_version(FileInfo { + version_id: Some(Uuid::from_u128(1)), + data_dir: Some(Uuid::from_u128(2)), + mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture modification time")), + size: 7, + parts: vec![rustfs_filemeta::ObjectPartInfo { + number: 1, + size: 7, + actual_size: 7, + ..Default::default() + }], + erasure: rustfs_filemeta::ErasureInfo { + algorithm: "ReedSolomon".to_string(), + data_blocks: 2, + parity_blocks: 2, + block_size: 1_048_576, + index: 1, + distribution: vec![1, 2, 3, 4], + ..Default::default() + }, + metadata: HashMap::from([("etag".to_string(), "legacy-etag".to_string())]), + ..Default::default() + }) + .expect("encode source object fixture"); + let (index, mut version) = metadata.find_version(Some(Uuid::from_u128(1))).expect("fixture version"); + let object = version.object.as_mut().expect("fixture object metadata"); + for (suffix, value) in [ + ("transition-status", b"complete".as_slice()), + ("transition-tier", b"WARM".as_slice()), + ("transitioned-object", b"archive/object".as_slice()), + ] { + object.meta_sys.insert(format!("x-minio-internal-{suffix}"), value.to_vec()); + } + if let Some(value) = version_bytes { + object + .meta_sys + .insert(format!("x-minio-internal-{SUFFIX_TRANSITIONED_VERSION_ID}"), value.to_vec()); + } + if let Some(value) = state { + object + .meta_sys + .insert(format!("x-minio-internal-{SUFFIX_TRANSITIONED_VERSION_STATE}"), value.to_vec()); + } + metadata.versions[index].header = version.header(); + metadata.versions[index].meta = version.marshal_msg().expect("encode legacy object version"); + metadata.marshal_msg().expect("encode legacy xl.meta") + } + + #[test] + fn raw_minio_and_rustfs_fixtures_preserve_legacy_provenance_without_backfill() { + for raw_version in [ + None, + Some(b"".as_slice()), + Some(b"opaque-version".as_slice()), + Some(Uuid::from_u128(9).as_bytes().as_slice()), + ] { + let bytes = legacy_metadata_fixture(raw_version, None); + let before = bytes.clone(); + let inspected = inspect_copy(&bytes, 0, "bucket", "object", Some(Uuid::from_u128(1))) + .expect("legacy metadata should decode") + .expect("legacy version should exist"); + assert_eq!(inspected.file_info.transition_version_state, TransitionVersionState::Unknown); + assert!(inspected.representation.state_aliases.is_empty()); + assert_eq!(inspected.representation.version_aliases.len(), usize::from(raw_version.is_some())); + assert_eq!(inspected.representation.metadata_digest, digest_hex(&before)); + assert_eq!(bytes, before, "inspection must not rewrite legacy bytes"); + } + } + + #[test] + fn corrupt_legacy_bytes_cannot_become_an_unversioned_candidate() { + for raw_version in [ + b"bad\nversion".as_slice(), + b"\xff\xfe".as_slice(), + Uuid::nil().as_bytes().as_slice(), + ] { + let bytes = legacy_metadata_fixture(Some(raw_version), None); + assert!(matches!( + inspect_copy(&bytes, 0, "bucket", "object", Some(Uuid::from_u128(1))), + Err(LegacyTransitionStateReconcileError::Corrupt(_)) + )); + } + let bytes = legacy_metadata_fixture(Some(b""), Some(b"unknown")); + let inspected = inspect_copy(&bytes, 0, "bucket", "object", Some(Uuid::from_u128(1))) + .expect("explicit unknown is structurally decodable") + .expect("fixture version"); + assert_eq!(inspected.file_info.transition_version_state, TransitionVersionState::Unknown); + assert!( + !inspected.representation.state_aliases.is_empty(), + "explicit unknown must remain distinct from legacy absence" + ); + } + + #[test] + fn selector_requires_explicit_non_nil_local_version_identity() { + for version_id in ["", "00000000-0000-0000-0000-000000000000", "not-a-version"] { + let selector = LegacyTransitionStateReconcileSelector { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: version_id.to_string(), + }; + assert!(matches!( + selector.canonicalize(), + Err(LegacyTransitionStateReconcileError::InvalidSelector(_)) + )); + } + assert!( + LegacyTransitionStateReconcileSelector { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: "null".to_string(), + } + .canonicalize() + .is_ok() + ); + } + + #[test] + fn digest_binds_source_sets_and_target() { + let source = LegacyTransitionStateSource { + bucket_incarnation: Uuid::from_u128(1).to_string(), + topology_generation: Some("topology-a".to_string()), + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: "null".to_string(), + data_dir: Uuid::from_u128(2).to_string(), + modification_time_unix_nanos: 1, + size: 1, + etag: "etag".to_string(), + transition_status: TRANSITION_COMPLETE.to_string(), + tier: "WARM".to_string(), + remote_object: "remote/object".to_string(), + }; + let sets = vec![LegacyTransitionStateSetRepresentation { + pool_index: 0, + set_index: 0, + total_copies: 1, + available_copies: 1, + copies: vec![], + }]; + let target = LegacyTransitionStateTarget { + state: TransitionVersionState::KnownDisabled, + remote_version: None, + destination_id: "00".repeat(32), + tier_generation: 1, + }; + let digest = response_digest(&source, &sets, &target).expect("valid source digest"); + let mut changed = source; + changed.remote_object.push_str("-changed"); + assert_ne!(digest, response_digest(&changed, &sets, &target).expect("changed source digest")); + } + + #[test] + fn nonempty_legacy_remote_version_must_match_live_candidate() { + let mut file_info = FileInfo { + transition_version: Some("remote-version-a".to_string()), + ..Default::default() + }; + let mut target = LegacyTransitionStateTarget { + state: TransitionVersionState::Exact, + remote_version: Some("remote-version-b".to_string()), + destination_id: "00".repeat(32), + tier_generation: 1, + }; + assert!(!legacy_remote_version_matches_target(&file_info, &target)); + target.remote_version = Some("remote-version-a".to_string()); + assert!(legacy_remote_version_matches_target(&file_info, &target)); + file_info.transition_version = None; + assert!(legacy_remote_version_matches_target(&file_info, &target)); + } + + #[test] + fn missing_state_accepts_only_empty_utf8_or_non_nil_raw_uuid_version_values() { + let key = format!("x-minio-internal-{SUFFIX_TRANSITIONED_VERSION_ID}"); + let mut metadata = HashMap::from([(key.clone(), Vec::new())]); + validate_remote_version_bytes(&metadata).expect("empty MinIO value is preserved legacy provenance"); + + metadata.insert(key.clone(), Uuid::from_u128(1).as_bytes().to_vec()); + validate_remote_version_bytes(&metadata).expect("non-nil historical RustFS UUID is valid provenance"); + + metadata.insert(key.clone(), Uuid::nil().as_bytes().to_vec()); + assert!(matches!( + validate_remote_version_bytes(&metadata), + Err(LegacyTransitionStateReconcileError::Corrupt(_)) + )); + + metadata.insert(key, vec![0xff, 0xfe]); + assert!(matches!( + validate_remote_version_bytes(&metadata), + Err(LegacyTransitionStateReconcileError::Corrupt(_)) + )); + } + + #[test] + fn raw_copy_quorum_cannot_be_lowered_by_embedded_erasure_geometry() { + let undersized = FileInfo { + erasure: rustfs_filemeta::ErasureInfo { + data_blocks: 1, + parity_blocks: 0, + ..Default::default() + }, + ..Default::default() + }; + let balanced = FileInfo { + erasure: rustfs_filemeta::ErasureInfo { + data_blocks: 4, + parity_blocks: 4, + ..Default::default() + }, + ..Default::default() + }; + + assert_eq!(required_reconcile_copy_quorum(&[undersized], 5), 5); + assert_eq!(required_reconcile_copy_quorum(&[balanced], 4), 5); + assert_eq!(required_reconcile_copy_quorum(&[], 5), 5); + + let old_pool = FileInfo { + erasure: rustfs_filemeta::ErasureInfo { + data_blocks: 8, + parity_blocks: 4, + ..Default::default() + }, + ..Default::default() + }; + let new_pool = FileInfo { + erasure: rustfs_filemeta::ErasureInfo { + data_blocks: 2, + parity_blocks: 2, + ..Default::default() + }, + ..Default::default() + }; + assert_eq!(required_reconcile_copy_quorum(&[old_pool], 7), 8); + assert_eq!(required_reconcile_copy_quorum(&[new_pool], 3), 3); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index de64dbbcf..ea56b5f8e 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -18,6 +18,7 @@ mod config_boundary; pub mod core; mod durable_namespace; pub mod evaluator; +pub mod legacy_transition_state_reconcile; pub mod manual_transition_job; mod metadata_boundary; pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, get_lifecycle_config}; diff --git a/crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs b/crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs index fcd886ca5..8e9057931 100644 --- a/crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs +++ b/crates/ecstore/src/bucket/lifecycle/runtime_boundary.rs @@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken; use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, TransitionState}; use crate::runtime::sources; +pub(crate) use crate::runtime::sources::ScannerDirtyUsageMutationSource; use crate::services::tier::tier::TierConfigMgr; use crate::store::ECStore; @@ -54,3 +55,7 @@ pub(crate) fn deployment_id() -> Option { pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option { sources::bucket_lifecycle_config(bucket).await } + +pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) { + sources::notify_scanner_dirty_usage_mutation(bucket, object, source); +} diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index c20825612..8f39640c5 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -248,10 +248,9 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) - } pub async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { - // The metadata system is inherently per-store (it holds the store handle - // and that store's bucket cache), so it lives on the store's own instance - // context (backlog#1052 S3) — a second instance initializes its own cell - // instead of panicking on the process-global one. + // The metadata system is inherently per-store, so it lives on the store's + // own instance context (backlog#1052 S3). It resolves the store through a + // weak handle so the context cache cannot keep the store and disks alive. let instance_ctx = api.ctx.clone(); let is_dist_erasure = instance_ctx.is_dist_erasure().await; @@ -317,18 +316,22 @@ fn start_refresh_buckets_metadata_loop(sys: Arc>) { warn!("bucket metadata refresh loop skipped because background cancellation token is not initialized"); return; }; + let sys = Arc::downgrade(&sys); tokio::spawn(async move { refresh_buckets_metadata_loop(sys, cancel_token).await; }); } -async fn refresh_buckets_metadata_loop(sys: Arc>, cancel_token: CancellationToken) { +async fn refresh_buckets_metadata_loop(sys: Weak>, cancel_token: CancellationToken) { loop { if !wait_refresh_interval_or_cancel(&cancel_token, BUCKET_METADATA_REFRESH_INTERVAL).await { break; } - refresh_buckets_metadata_once(sys.clone()).await; + let Some(sys) = sys.upgrade() else { + break; + }; + refresh_buckets_metadata_once(sys).await; } } @@ -455,7 +458,7 @@ pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceCont pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option> { let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?; - Some(sys.read().await.api.clone()) + sys.read().await.object_store_if_live() } pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result> { @@ -475,7 +478,7 @@ pub(crate) async fn get_config_from_disk_with_presence_in( bucket: &str, ) -> Result<(BucketMetadata, bool)> { let sys = bucket_metadata_sys_of(ctx)?; - let api = sys.read().await.api.clone(); + let api = sys.read().await.object_store(); load_bucket_metadata_parse_with_presence(api, bucket, true).await } @@ -738,7 +741,7 @@ pub async fn acquire_scanner_bucket_incarnation_fence( ) -> Result { super::utils::check_valid_bucket_name(bucket)?; let sys = get_bucket_metadata_sys()?; - if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() { + if expected_owner_id.is_nil() || sys.read().await.object_store().id != expected_owner_id || expected_incarnation_id.is_nil() { return Err(Error::other("scanner bucket incarnation owner does not match")); } acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await @@ -751,7 +754,7 @@ async fn acquire_config_write_guard_with_migration( migrate: bool, ) -> Result { let metadata_sys = sys.read().await.clone(); - let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?; + let lifecycle_guard = metadata_sys.object_store().acquire_bucket_lifecycle_read_lock(bucket).await?; // Legacy buckets are migrated while the lifecycle fence prevents a // same-name replacement. The second read under the write transaction is @@ -782,7 +785,7 @@ async fn acquire_config_write_guard_with_migration( "bucket config existence transaction validation", async { match metadata_sys - .api + .object_store() .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) .await { @@ -802,7 +805,7 @@ async fn acquire_config_write_guard_with_migration( Some(&transaction_guard), bucket, "bucket config incarnation transaction validation", - load_bucket_incarnation(metadata_sys.api.clone(), bucket), + load_bucket_incarnation(metadata_sys.object_store(), bucket), ), ) .await? @@ -1461,7 +1464,7 @@ pub struct BucketMetadataSys { /// Physically missing names are TTL-bounded to limit memory under bogus /// name floods while avoiding repeated namespace and erasure reads. missing_buckets: moka::future::Cache, - api: Arc, + api: Weak, } impl BucketMetadataSys { @@ -1489,12 +1492,17 @@ impl BucketMetadataSys { .max_capacity(MISSING_BUCKET_MAX_ENTRIES) .time_to_live(MISSING_BUCKET_TTL) .build(), - api, + api: Arc::downgrade(&api), } } pub(crate) fn object_store(&self) -> Arc { - self.api.clone() + self.object_store_if_live() + .expect("bucket metadata object store should still be live") + } + + fn object_store_if_live(&self) -> Option> { + self.api.upgrade() } fn metadata_publish_lock(&self, bucket: &str) -> Arc> { @@ -1549,7 +1557,7 @@ impl BucketMetadataSys { ) -> Result { await_bucket_namespace_operation(Some(namespace_guard), bucket, operation, async { match self - .api + .object_store() .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) .await { @@ -1566,7 +1574,7 @@ impl BucketMetadataSys { } async fn init_internal(&self, buckets: Vec) -> Result<()> { let count = self - .api + .object_store() .pools .iter() .map(|pool| pool.disk_set.len()) @@ -1598,7 +1606,7 @@ impl BucketMetadataSys { let mut futures = Vec::new(); for bucket in buckets.iter() { - let api = self.api.clone(); + let api = self.object_store(); let bucket = bucket.clone(); futures.push(async move { sleep(Duration::from_millis(30)).await; @@ -1644,7 +1652,9 @@ impl BucketMetadataSys { let bucket = bucket.clone(); futures.push(async move { sleep(Duration::from_millis(30)).await; - let api = sys.read().await.api.clone(); + let Some(api) = sys.read().await.object_store_if_live() else { + return Ok(()); + }; let namespace_lock = api.new_ns_lock(&bucket, &bucket).await?; let namespace_guard = namespace_lock .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) @@ -1681,9 +1691,13 @@ impl BucketMetadataSys { expected: Option<&Arc>, namespace_guard: &rustfs_lock::NamespaceLockGuard, ) -> Result<()> { - if !self - .bucket_exists(bucket, namespace_guard, "bucket metadata existence check") - .await? + if !await_bucket_namespace_operation( + Some(namespace_guard), + bucket, + "bucket metadata heal existence check", + self.object_store().bucket_exists_for_heal(bucket), + ) + .await? { if matches!(mode, MetadataLoadMode::Refresh) { let _publish_guard = self @@ -1705,7 +1719,7 @@ impl BucketMetadataSys { Some(namespace_guard), bucket, "bucket metadata heal", - self.api.heal_bucket( + self.object_store().heal_bucket( bucket, &HealOpts { recreate: true, @@ -1719,7 +1733,7 @@ impl BucketMetadataSys { Some(namespace_guard), bucket, "bucket metadata load", - load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true), + load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true), ) .await?; match mode { @@ -1899,7 +1913,7 @@ impl BucketMetadataSys { // (backlog#1052 S7). Reading from the ambient handle instead made the // read and the write of a single read-modify-write able to target // different instances. - let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, parse)).await?; + let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, parse)).await?; if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id { return Err(Error::BucketNotFound(bucket.to_string())); } @@ -1938,7 +1952,7 @@ impl BucketMetadataSys { where F: FnOnce(&BucketMetadata) -> Result> + Send, { - let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.api.clone(), bucket, true)).await?; + let mut bm = Box::pin(Self::load_bucket_metadata_for_update(self.object_store(), bucket, true)).await?; if !bm.bucket_incarnation_sidecar || bm.bucket_incarnation_id != expected_incarnation_id { return Err(Error::BucketNotFound(bucket.to_string())); } @@ -1989,7 +2003,7 @@ impl BucketMetadataSys { /// server's metadata never leaks into the ambient (first) instance. pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> { let mut bm = bm; - bm.save_with_store(self.api.clone()).await?; + bm.save_with_store(self.object_store()).await?; self.set(bm.name.clone(), Arc::new(bm)).await; @@ -1997,8 +2011,8 @@ impl BucketMetadataSys { } async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> { - bm.save_with_store(self.api.clone()).await?; - save_bucket_incarnation(self.api.clone(), &bm.name, bm.bucket_incarnation_id).await?; + bm.save_with_store(self.object_store()).await?; + save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?; bm.bucket_incarnation_sidecar = true; self.set(bm.name.clone(), Arc::new(bm)).await; Ok(()) @@ -2015,7 +2029,7 @@ impl BucketMetadataSys { return Err(Error::other("errInvalidArgument")); } - load_bucket_metadata(self.api.clone(), bucket).await + load_bucket_metadata(self.object_store(), bucket).await } /// Reload persisted metadata under the bucket namespace generation fence. @@ -2029,7 +2043,7 @@ impl BucketMetadataSys { return Err(Error::other("errInvalidArgument")); } - let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?; + let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?; let namespace_guard = namespace_lock .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) .await?; @@ -2052,7 +2066,7 @@ impl BucketMetadataSys { Some(namespace_guard), bucket, "peer bucket metadata load", - load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true), + load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true), ) .await?; if !persisted { @@ -2097,11 +2111,11 @@ impl BucketMetadataSys { #[cfg(test)] self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let lock = self.api.new_ns_lock(bucket, bucket).await?; + let lock = self.object_store().new_ns_lock(bucket, bucket).await?; let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?; #[cfg(test)] if self.lazy_load_lock_probe.load(std::sync::atomic::Ordering::Relaxed) { - let competing = self.api.new_ns_lock(bucket, bucket).await?; + let competing = self.object_store().new_ns_lock(bucket, bucket).await?; assert!( competing.get_write_lock(Duration::from_millis(20)).await.is_err(), "lazy metadata IO must start while the bucket namespace read lock is held" @@ -2111,7 +2125,7 @@ impl BucketMetadataSys { Some(&guard), bucket, "lazy bucket metadata load", - Box::pin(load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true)), + Box::pin(load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true)), ) .await?; @@ -2123,7 +2137,7 @@ impl BucketMetadataSys { bucket, "lazy bucket metadata existence check", Box::pin(async { - self.api + self.object_store() .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) .await .map(|_| ()) @@ -2330,13 +2344,13 @@ impl BucketMetadataSys { async fn get_bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result { let transaction_lock = self - .api + .object_store() .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) .await?; let _transaction_guard = transaction_lock .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) .await?; - let incarnation_id = load_bucket_incarnation(self.api.clone(), bucket).await?; + let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?; if _transaction_guard.is_lock_lost() { return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}"))); } @@ -2372,7 +2386,7 @@ impl BucketMetadataSys { async fn migrate_legacy_metadata(&self, bucket: &str) -> Result { let transaction_lock = self - .api + .object_store() .new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket)) .await?; let _transaction_guard = transaction_lock @@ -2397,7 +2411,7 @@ impl BucketMetadataSys { return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}"))); } - let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?; + let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?; let namespace_guard = namespace_lock .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) .await?; @@ -2407,7 +2421,7 @@ impl BucketMetadataSys { bucket, "legacy bucket metadata existence check", async { - self.api + self.object_store() .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) .await }, @@ -2431,7 +2445,7 @@ impl BucketMetadataSys { Some(&namespace_guard), bucket, "legacy bucket metadata confirmation", - load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true), + load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true), ) .await?; if persisted && !metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() { @@ -2453,20 +2467,20 @@ impl BucketMetadataSys { } #[cfg(test)] if self.legacy_migration_lock_probe.load(std::sync::atomic::Ordering::Relaxed) { - let competing = self.api.new_ns_lock(bucket, bucket).await?; + let competing = self.object_store().new_ns_lock(bucket, bucket).await?; assert!( competing.get_write_lock(Duration::from_millis(20)).await.is_err(), "bucket delete/recreate must not cross the legacy metadata migration fence" ); } - save_bucket_incarnation(self.api.clone(), bucket, metadata.bucket_incarnation_id).await?; + save_bucket_incarnation(self.object_store(), bucket, metadata.bucket_incarnation_id).await?; metadata.bucket_incarnation_sidecar = true; if !persisted { await_bucket_namespace_operation( Some(&namespace_guard), bucket, "legacy bucket metadata migration", - metadata.save_with_store(self.api.clone()), + metadata.save_with_store(self.object_store()), ) .await?; } @@ -2502,7 +2516,7 @@ impl BucketMetadataSys { return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}"))); } - let namespace_lock = self.api.new_ns_lock(bucket, bucket).await?; + let namespace_lock = self.object_store().new_ns_lock(bucket, bucket).await?; let namespace_guard = namespace_lock .get_read_lock(crate::set_disk::get_lock_acquire_timeout()) .await?; @@ -2511,8 +2525,11 @@ impl BucketMetadataSys { bucket, "bucket metadata snapshot existence check", async { - self.api - .get_bucket_info_from_sets(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default()) + self.object_store() + .get_bucket_info_from_sets_at_read_quorum( + bucket, + &crate::storage_api_contracts::bucket::BucketOptions::default(), + ) .await }, ) @@ -2527,7 +2544,7 @@ impl BucketMetadataSys { Some(&namespace_guard), bucket, "bucket metadata authoritative snapshot", - load_bucket_metadata_parse_with_presence(self.api.clone(), bucket, true), + load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true), ) .await?; if persisted { @@ -3691,7 +3708,7 @@ mod tests { let mut stale = BucketMetadata::new("recreated-bucket"); stale.policy_config_json = b"old-generation".to_vec(); let namespace_lock = sys - .api + .object_store() .new_ns_lock("recreated-bucket", "recreated-bucket") .await .expect("namespace lock should be created"); diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index bb9e002d0..031fc9d32 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -92,3 +92,6 @@ pub use replication_target_boundary::SsecPassthroughCapability; pub use replication_target_boundary::VersionIdentityCapability; pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity}; pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge; +pub use runtime_boundary::{ + ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer, +}; diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 1a8094465..bb768e974 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -3843,6 +3843,13 @@ async fn persist_replication_state_if_current( match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await { Ok(updated) => { *object_info = updated; + if mode == ReplicationStatusWritebackMode::Update { + runtime_sources::notify_scanner_dirty_usage_mutation( + &roi.bucket, + &roi.name, + runtime_sources::ScannerDirtyUsageMutationSource::Replication, + ); + } Ok(ReplicationStatePersistOutcome::Updated) } Err(Error::PreconditionFailed) => Ok(ReplicationStatePersistOutcome::Superseded), diff --git a/crates/ecstore/src/bucket/replication/runtime_boundary.rs b/crates/ecstore/src/bucket/replication/runtime_boundary.rs index d930c6119..36df30fc3 100644 --- a/crates/ecstore/src/bucket/replication/runtime_boundary.rs +++ b/crates/ecstore/src/bucket/replication/runtime_boundary.rs @@ -19,6 +19,9 @@ use super::replication_pool::DynReplicationPool; use super::replication_state::ReplicationStats; use super::replication_storage_boundary::ReplicationObjectStore; use crate::runtime::sources; +pub use crate::runtime::sources::{ + ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer, +}; pub(crate) fn object_store_handle() -> Option> { sources::object_store_handle() @@ -43,3 +46,7 @@ pub(crate) fn replication_runtime_initialized() -> bool { pub(crate) fn bucket_monitor() -> Option> { sources::bucket_monitor() } + +pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) { + sources::notify_scanner_dirty_usage_mutation(bucket, object, source); +} diff --git a/crates/ecstore/src/cluster/rpc/client.rs b/crates/ecstore/src/cluster/rpc/client.rs index c4bf786f9..b6a3bda17 100644 --- a/crates/ecstore/src/cluster/rpc/client.rs +++ b/crates/ecstore/src/cluster/rpc/client.rs @@ -285,6 +285,20 @@ fn peer_replay_state(audience: &str) -> PeerReplayState { .unwrap_or_default() } +pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> { + let uri = addr + .parse::() + .map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?; + let audience = uri + .authority() + .map(|authority| normalize_tonic_rpc_audience(authority.as_str())) + .ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??; + if let Ok(mut states) = PEER_REPLAY_STATES.lock() { + states.remove(&audience); + } + Ok(()) +} + fn apply_peer_replay_response( audience: String, sent_state: PeerReplayState, @@ -619,6 +633,13 @@ mod tests { .remove(audience); } + fn set_peer_capability(audience: &str, state: PeerReplayState) { + PEER_REPLAY_STATES + .lock() + .expect("peer capability cache lock must not be poisoned") + .insert(audience.to_string(), state); + } + fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> { let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest { args: "canonical mutation request".to_string(), @@ -1090,6 +1111,23 @@ mod tests { clear_peer_capability(audience); } + #[test] + fn clear_peer_replay_state_for_addr_removes_normalized_audience() { + let audience = "clear-peer-replay-state-test:9000"; + let boot_epoch = Uuid::new_v4(); + set_peer_capability( + audience, + PeerReplayState { + boot_epoch: Some(boot_epoch), + cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }), + }, + ); + + clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state"); + + assert_eq!(peer_replay_state(audience), PeerReplayState::default()); + } + #[test] fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() { ensure_test_rpc_secret(); diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 8b992985f..ea3e361a6 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -13,8 +13,9 @@ // limitations under the License. use crate::cluster::rpc::client::{ - AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client, - is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client, + AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status, + gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle, + node_service_time_out_client, tier_mutation_control_time_out_client, }; use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof}; use crate::error::{Error, Result}; @@ -50,10 +51,10 @@ use rustfs_protos::proto_gen::node_service::{ LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest, ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, - ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest, - SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest, - TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass, - TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient, + ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse, ScannerScopedDirtyUsageEntry, ServerInfoRequest, + SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, + TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, + TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient, }; pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS}; @@ -288,8 +289,30 @@ fn scanner_scoped_dirty_usage_ack_payload( Ok(payload) } -fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool { - activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false) +fn scanner_scoped_dirty_usage_ack_response_matches( + request: &ScannerScopedDirtyUsageAckRequest, + response: &ScannerScopedDirtyUsageAckResponse, +) -> bool { + let cleared_within_request = u64::try_from(request.entries.len()) + .is_ok_and(|entry_count| response.cleared <= entry_count && (!request.probe_only || response.cleared == 0)); + response.protocol_version == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_PROTOCOL_VERSION + && response.owner_id == request.owner_id + && response.instance_id == request.instance_id + && response.max_entries == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_ENTRIES + && response.max_request_bytes == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES + && cleared_within_request +} + +fn scanner_scoped_dirty_usage_ack_reconciled( + activity: &ScannerPeerActivity, + expected_instance_id: &str, + expected_generation: u64, +) -> bool { + activity.instance_id == expected_instance_id + && activity.dirty_usage_pending == Some(false) + && activity + .dirty_usage_generation + .is_some_and(|generation| generation >= expected_generation) } fn scanner_instance_id_is_valid(instance_id: &str) -> bool { @@ -544,6 +567,16 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8]) .map_err(|_| Error::other("peer returned an invalid heal control response proof")) } +fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool { + matches!( + err, + Error::Io(io_err) + if embedded_tonic_status(io_err).is_some_and(|status| { + status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token" + }) + ) +} + fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result { let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?; if topology_member != expected_member { @@ -1720,45 +1753,72 @@ impl PeerRestClient { return Err(Error::other("heal control command exceeds size limit")); } let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command); - self.finalize_result( - async { - let mut client = self - .get_heal_control_client() - .await? - .max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE) - .max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE); - let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command) - .map_err(|_| Error::other("heal control request length cannot be represented"))?; - let mut request = Request::new(HealControlRequest { - version, - topology_fingerprint: topology_fingerprint.clone(), - command: command.clone().into(), - }); - request.set_timeout(rustfs_protos::heal_control_execution_timeout()); - set_tonic_canonical_body_digest(&mut request, &canonical_body)?; - let response = client.heal_control(request).await?.into_inner(); - if !response.success { - return Err(Error::other( - response - .error_info - .unwrap_or_else(|| "peer heal control failed without an error".to_string()), - )); - } - if !capability_probe { - let canonical_response = rustfs_protos::canonical_heal_control_response_body( - version, - &topology_fingerprint, - &command, - &response.result, - ) + let result = self + .heal_control_once(version, &topology_fingerprint, &command, capability_probe) + .await; + if result + .as_ref() + .err() + .is_some_and(heal_control_auth_may_need_replay_scope_refresh) + { + self.prepare_heal_control_auth_retry().await; + return self + .finalize_result( + self.heal_control_once(version, &topology_fingerprint, &command, capability_probe) + .await, + ) + .await; + } + self.finalize_result(result).await + } + + async fn prepare_heal_control_auth_retry(&self) { + if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) { + debug!( + peer = %self.grid_host, + error = %err, + "could not clear heal control replay state before retry" + ); + } + self.evict_connection().await; + } + + async fn heal_control_once( + &self, + version: u32, + topology_fingerprint: &str, + command: &[u8], + capability_probe: bool, + ) -> Result> { + let mut client = self + .get_heal_control_client() + .await? + .max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE) + .max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE); + let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command) + .map_err(|_| Error::other("heal control request length cannot be represented"))?; + let mut request = Request::new(HealControlRequest { + version, + topology_fingerprint: topology_fingerprint.to_string(), + command: command.to_vec().into(), + }); + request.set_timeout(rustfs_protos::heal_control_execution_timeout()); + set_tonic_canonical_body_digest(&mut request, &canonical_body)?; + let response = client.heal_control(request).await?.into_inner(); + if !response.success { + return Err(Error::other( + response + .error_info + .unwrap_or_else(|| "peer heal control failed without an error".to_string()), + )); + } + if !capability_probe { + let canonical_response = + rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result) .map_err(|_| Error::other("heal control response length cannot be represented"))?; - validate_heal_control_response_proof(&canonical_response, &response.response_proof)?; - } - Ok(response.result.to_vec()) - } - .await, - ) - .await + validate_heal_control_response_proof(&canonical_response, &response.response_proof)?; + } + Ok(response.result.to_vec()) } /// Confirms that a peer supports the current heal-control coordination @@ -2217,13 +2277,7 @@ impl PeerRestClient { let body = canonical_scoped_dirty_usage_response(&canonical, &response) .map_err(|_| Error::other("scoped dirty usage capability response is too large"))?; verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?; - if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION - || response.owner_id != payload.owner_id - || response.instance_id != payload.instance_id - || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES - || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES - || response.cleared != 0 - { + if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) { return Err(Error::other("scoped dirty usage capability response does not match request")); } if !response.supported { @@ -2244,6 +2298,7 @@ impl PeerRestClient { entries: Vec, ) -> Result { use rustfs_protos::scoped_dirty_usage::*; + let expected_generation = entries.iter().map(|entry| entry.generation).max().unwrap_or(0); let payloads = scanner_scoped_dirty_usage_ack_payloads(owner_id, instance_id.clone(), false, entries)?; let ack_attempt = async { let mut client = super::client::scanner_control_time_out_client( @@ -2259,13 +2314,7 @@ impl PeerRestClient { let body = canonical_scoped_dirty_usage_response(&canonical, &response) .map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?; verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?; - if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION - || response.owner_id != payload.owner_id - || response.instance_id != payload.instance_id - || response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES - || response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES - || !response.supported - { + if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) || !response.supported { return Err(Error::other("scoped dirty usage acknowledgement response does not match request")); } } @@ -2297,7 +2346,9 @@ impl PeerRestClient { .await; } match self.scanner_scoped_dirty_usage_activity_confirmation().await { - Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id) => Ok(activity), + Ok(activity) if scanner_scoped_dirty_usage_ack_reconciled(&activity, &instance_id, expected_generation) => { + Ok(activity) + } _ => Err(err), } } @@ -3080,32 +3131,104 @@ mod tests { } } + #[test] + fn scanner_scoped_dirty_usage_ack_response_bounds_cleared_entries_to_request() { + use rustfs_protos::scoped_dirty_usage::{ + SCOPED_DIRTY_USAGE_MAX_ENTRIES, SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, SCOPED_DIRTY_USAGE_PROTOCOL_VERSION, + }; + + let mut request = scanner_scoped_dirty_usage_ack_payload( + "33333333-3333-3333-3333-333333333333", + "0123456789abcdef0123456789abcdef", + false, + vec![ + ScannerScopedDirtyUsageEntry { + bucket: "archive".to_string(), + bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(), + generation: 3, + }, + ScannerScopedDirtyUsageEntry { + bucket: "photos".to_string(), + bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(), + generation: 7, + }, + ], + ) + .expect("two ordered entries should form a valid scoped ACK request"); + let mut response = ScannerScopedDirtyUsageAckResponse { + protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION, + owner_id: request.owner_id.clone(), + instance_id: request.instance_id.clone(), + supported: true, + max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES, + max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, + cleared: 1, + response_proof: Bytes::new(), + }; + + assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response)); + response.cleared = 2; + assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response)); + response.cleared = 3; + assert!( + !scanner_scoped_dirty_usage_ack_response_matches(&request, &response), + "a peer cannot clear more entries than the signed request contains" + ); + + request.probe_only = true; + response.cleared = 1; + assert!( + !scanner_scoped_dirty_usage_ack_response_matches(&request, &response), + "a capability probe cannot report a mutation" + ); + response.cleared = 0; + assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response)); + } + #[test] fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() { - let activity = |instance_id: &str, pending| ScannerPeerActivity { + let activity = |instance_id: &str, generation, pending| ScannerPeerActivity { instance_id: instance_id.to_string(), namespace_generation: 1, maintenance_generation: 1, protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, topology_digest: Some([1; 32]), data_movement_active: Some(false), - dirty_usage_generation: Some(9), + dirty_usage_generation: generation, dirty_usage_pending: pending, movement_generation: Some(1), publication_blocked: Some(false), }; assert!(scanner_scoped_dirty_usage_ack_reconciled( - &activity("0123456789abcdef0123456789abcdef", Some(false)), - "0123456789abcdef0123456789abcdef" + &activity("0123456789abcdef0123456789abcdef", Some(9), Some(false)), + "0123456789abcdef0123456789abcdef", + 9 + )); + assert!(scanner_scoped_dirty_usage_ack_reconciled( + &activity("0123456789abcdef0123456789abcdef", Some(10), Some(false)), + "0123456789abcdef0123456789abcdef", + 9 )); assert!(!scanner_scoped_dirty_usage_ack_reconciled( - &activity("0123456789abcdef0123456789abcdef", Some(true)), - "0123456789abcdef0123456789abcdef" + &activity("0123456789abcdef0123456789abcdef", Some(8), Some(false)), + "0123456789abcdef0123456789abcdef", + 9 )); assert!(!scanner_scoped_dirty_usage_ack_reconciled( - &activity("fedcba9876543210fedcba9876543210", Some(false)), - "0123456789abcdef0123456789abcdef" + &activity("0123456789abcdef0123456789abcdef", None, Some(false)), + "0123456789abcdef0123456789abcdef", + 9 + )); + assert!(!scanner_scoped_dirty_usage_ack_reconciled( + &activity("0123456789abcdef0123456789abcdef", Some(9), Some(true)), + "0123456789abcdef0123456789abcdef", + 9 + )); + assert!(!scanner_scoped_dirty_usage_ack_reconciled( + &activity("fedcba9876543210fedcba9876543210", Some(9), Some(false)), + "0123456789abcdef0123456789abcdef", + 9 )); } @@ -3728,6 +3851,22 @@ mod tests { } } + #[test] + fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() { + assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from( + tonic::Status::unauthenticated("No valid auth token") + ))); + assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from( + tonic::Status::permission_denied("bad signature") + ))); + assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from( + tonic::Status::unauthenticated("application rejected heal control") + ))); + assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other( + "Io error: code: 'Unauthenticated', message: \"No valid auth token\"" + ))); + } + #[test] fn peer_rest_client_network_classifier_keeps_slow_peers_online() { // The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index db001db79..8635b57ac 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -152,6 +152,7 @@ pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3; const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50); const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100; const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1); +const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts"; const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests"; const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2"; @@ -2606,6 +2607,24 @@ where usize::try_from(size).unwrap_or_default() } +fn is_decommission_set_local_usage_cache(bucket: &str, object: &str) -> bool { + if bucket != RUSTFS_META_BUCKET { + return false; + } + let Some(path) = object + .strip_prefix(BUCKET_META_PREFIX) + .and_then(|path| path.strip_prefix('/')) + else { + return false; + }; + let name = match path.rsplit_once('/') { + Some((bucket, name)) if !bucket.is_empty() && !bucket.contains('/') && bucket != "." && bucket != ".." => name, + Some(_) => return false, + None => path, + }; + name.strip_suffix(".bkp").unwrap_or(name) == DATA_USAGE_CACHE_NAME +} + fn with_decommission_entry_context(stage: &str, bucket: &str, object: &str, err: E) -> Error { Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}")) } @@ -5510,7 +5529,7 @@ where meta, revision, committed, - .. + previous, } => { observation["state"] = serde_json::json!("valid"); observation["committed"] = serde_json::json!(committed); @@ -5522,6 +5541,16 @@ where observation["pool_count"] = serde_json::json!(meta.pools.len()); observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical))); observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw))); + match pool_meta_test_pools_sha256(meta) { + Ok(digest) => observation["persisted_pools_sha256"] = serde_json::json!(digest), + Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()), + } + if let Some(previous) = previous { + match pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical) { + Ok(summary) => observation["previous"] = summary, + Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()), + } + } } PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"), PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"), @@ -6046,18 +6075,223 @@ pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) { let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); } -async fn save_pool_meta_object_cas( - pool: Arc, - object: &str, - data: Vec, - token: &PoolMetaCasToken, - fence: &PoolMetaPersistenceFence<'_>, - phase: &'static str, - transaction_arm: &mut PoolMetaTransactionArm, -) -> Result -where - S: EcstoreObjectIO, -{ +#[cfg(feature = "e2e-test-hooks")] +fn pool_meta_test_pools_sha256(meta: &PoolMeta) -> Result { + let pools = meta.pools.iter().map(PersistedPoolStatus::from).collect::>(); + Ok(rustfs_utils::crypto::hex(Sha256::digest(serde_json::to_vec(&pools)?))) +} + +#[cfg(feature = "e2e-test-hooks")] +fn pool_meta_test_snapshot(meta: &PoolMeta, revision: PoolMetaRevision, canonical: &[u8]) -> Result { + Ok(serde_json::json!({ + "version": revision.version, "cluster_id": revision.cluster_id, "epoch": revision.epoch, + "generation": revision.generation, "transaction_id": revision.transaction_id, + "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(canonical)), + "persisted_pools_sha256": pool_meta_test_pools_sha256(meta)?, + })) +} + +#[cfg(feature = "e2e-test-hooks")] +#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +enum PoolMetaPhaseBarrierCase { + PrepareSubset, + PreparedAll, + CommitOne, + BeforePublish, +} + +#[cfg(feature = "e2e-test-hooks")] +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PoolMetaPhaseBarrierArm { + nonce: uuid::Uuid, + case: PoolMetaPhaseBarrierCase, +} + +#[cfg(feature = "e2e-test-hooks")] +struct PoolMetaPhaseBarrier { + directory: std::path::PathBuf, + arm: PoolMetaPhaseBarrierArm, + transaction_id: uuid::Uuid, + event_gate: tokio::sync::Mutex<()>, + blocked_pools: AtomicUsize, + completed_writes: AtomicUsize, + ready_emitted: AtomicBool, +} + +#[cfg(feature = "e2e-test-hooks")] +impl PoolMetaPhaseBarrier { + async fn bind( + previous: &PoolMetaCommittedCandidate, + candidate: &PoolMeta, + revision: PoolMetaRevision, + durable: &[u8], + ) -> Result> { + let Some(directory) = std::env::var_os("RUSTFS_E2E_POOL_META_BARRIER_DIR").map(std::path::PathBuf::from) else { + return Ok(None); + }; + Self::bind_in_directory(directory, previous, candidate, revision, durable).await + } + + async fn bind_in_directory( + directory: std::path::PathBuf, + previous: &PoolMetaCommittedCandidate, + candidate: &PoolMeta, + revision: PoolMetaRevision, + durable: &[u8], + ) -> Result> { + let data = match tokio::fs::read(directory.join("arm.json")).await { + Ok(data) => data, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(err.into()), + }; + let arm: PoolMetaPhaseBarrierArm = serde_json::from_slice(&data)?; + if arm.nonce.is_nil() || !directory.is_absolute() { + return Err(Error::other( + "pool metadata test barrier requires an absolute directory and non-nil nonce", + )); + } + let transaction_id = revision + .transaction_id + .ok_or_else(|| Error::other("pool metadata test barrier requires a V3 transaction"))?; + // A new revision can still carry the same persisted pool state. Leave + // the external arm available until recovery can distinguish P from G. + if pool_meta_test_pools_sha256(&previous.meta)? == pool_meta_test_pools_sha256(candidate)? { + return Ok(None); + } + // One external arm binds one attempt, including if its CAS subsequently retries. + let claim = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join("claimed")) + .await; + match claim { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None), + Err(err) => return Err(err.into()), + } + let barrier = Self { + directory, + arm, + transaction_id, + event_gate: tokio::sync::Mutex::new(()), + blocked_pools: AtomicUsize::new(0), + completed_writes: AtomicUsize::new(0), + ready_emitted: AtomicBool::new(false), + }; + barrier + .observe(serde_json::json!({ + "kind": "armed", + "previous": pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical)?, + "candidate": pool_meta_test_snapshot(candidate, revision, durable)?, + })) + .await?; + Ok(Some(barrier)) + } + + async fn observe(&self, mut event: serde_json::Value) -> Result<()> { + use tokio::io::AsyncWriteExt as _; + event["nonce"] = serde_json::json!(self.arm.nonce); + event["pid"] = serde_json::json!(std::process::id()); + event["transaction_id"] = serde_json::json!(self.transaction_id); + event["case"] = serde_json::json!(self.arm.case); + let mut line = serde_json::to_vec(&event)?; + line.push(b'\n'); + let _event_guard = self.event_gate.lock().await; + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.directory.join("events.jsonl")) + .await?; + file.write_all(&line).await?; + file.flush().await?; + Ok(()) + } + + async fn wait_for_release(&self) -> Result<()> { + tokio::time::timeout(std::time::Duration::from_secs(120), async { + loop { + match tokio::fs::metadata(self.directory.join("release")).await { + Ok(_) => return Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::from(err)), + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| Error::other("pool metadata test barrier release timed out"))? + } + + async fn ready(&self) -> Result<()> { + self.observe(serde_json::json!({"kind": "ready", "cutpoint": self.arm.case})) + .await?; + self.wait_for_release().await + } + + async fn ready_after_all_participants(&self) -> Result<()> { + let (blocked, completed) = match self.arm.case { + PoolMetaPhaseBarrierCase::PrepareSubset => (0b0101, 0b1010), + PoolMetaPhaseBarrierCase::CommitOne => (0b0111, 0b1000), + _ => return Ok(()), + }; + if self.blocked_pools.load(Ordering::SeqCst) == blocked + && self.completed_writes.load(Ordering::SeqCst) == completed + && !self.ready_emitted.swap(true, Ordering::SeqCst) + { + self.ready().await?; + } + Ok(()) + } + + async fn before_write(&self, phase: &'static str, pool: usize, data: &[u8]) -> Result<()> { + let blocked = match (self.arm.case, phase) { + (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas") => ![1, 3].contains(&pool), + (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas") => pool != 3, + _ => false, + }; + if blocked { + self.observe(serde_json::json!({ + "kind": "blocked", "phase": phase, "pool": pool, + "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)), + })) + .await?; + self.blocked_pools.fetch_or(1 << pool, Ordering::SeqCst); + self.ready_after_all_participants().await?; + self.wait_for_release().await?; + } + self.observe(serde_json::json!({ + "kind": "before-dispatch", "phase": phase, "pool": pool, + "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)), + })) + .await + } + + async fn after_write(&self, phase: &'static str, pool: usize, data: &[u8], outcome: &PoolMetaCasWriteOutcome) -> Result<()> { + self.observe(serde_json::json!({ + "kind": "after-cas", "phase": phase, "pool": pool, + "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)), + "ok": outcome.result.is_ok(), "tail_drained": outcome.result.is_ok(), + "etag": outcome.result.as_ref().ok().and_then(|info| info.etag.as_deref()), + "may_have_mutated": outcome.may_have_mutated, + })) + .await?; + if outcome.result.is_ok() { + match (self.arm.case, phase, pool) { + (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas", 1 | 3) + | (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas", 3) => { + self.completed_writes.fetch_or(1 << pool, Ordering::SeqCst); + self.ready_after_all_participants().await?; + } + _ => {} + } + } + Ok(()) + } +} + +fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMetaPersistenceFence<'_>) -> Result { fence.ensure_held()?; let mut opts = ObjectOptions { max_parity: true, @@ -6067,6 +6301,40 @@ where ..Default::default() }; fence.add_to_options(&mut opts); + Ok(opts) +} + +struct PoolMetaCasWriteOutcome { + result: Result, + source: Option>, + may_have_mutated: bool, +} + +impl PoolMetaCasWriteOutcome { + fn not_dispatched(err: Error) -> Self { + Self { + result: Err(err), + source: None, + may_have_mutated: false, + } + } +} + +async fn execute_pool_meta_object_cas( + pool: Arc, + object: &str, + data: Vec, + opts: ObjectOptions, + fence: &PoolMetaPersistenceFence<'_>, + phase: &'static str, +) -> PoolMetaCasWriteOutcome +where + S: EcstoreObjectIO, +{ + // A bounded phase can wait for another replica before this write is polled. + if let Err(err) = fence.ensure_held() { + return PoolMetaCasWriteOutcome::not_dispatched(err); + } #[cfg(feature = "e2e-test-hooks")] let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| { serde_json::json!({ @@ -6081,27 +6349,24 @@ where "no_lock": opts.no_lock, }) }); - // Cancellation can happen at the very first poll of the storage future. - // Arm before dispatch, but not during read/encode/fence preflight. - let previous_phase = transaction_arm.phase; - transaction_arm.phase = Some(phase); let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await; - if matches!(&result, Err(Error::PreconditionFailed)) { + let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed)); + if !may_have_mutated { record_pool_meta_stale_write_rejection(phase); - transaction_arm.phase = previous_phase; } + let mut source = None; let result = match result { Ok(object_info) => fence.ensure_held().map(|()| object_info), Err(err) => { - let source = Arc::new(err); - transaction_arm.source = Some(Arc::clone(&source)); - if matches!(source.as_ref(), Error::PreconditionFailed) { + let original = Arc::new(err); + source = Some(Arc::clone(&original)); + if matches!(original.as_ref(), Error::PreconditionFailed) { Err(Error::PreconditionFailed) } else { Err(Error::other(pool_metadata_error( crate::error::PoolMetadataFailure::TransactionUnknown, phase, - Some(source), + Some(original), ))) } } @@ -6120,7 +6385,127 @@ where observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string)); startup_cas_test_observe(observation); } - result + PoolMetaCasWriteOutcome { + result, + source, + may_have_mutated, + } +} + +async fn save_pool_meta_object_cas( + pool: Arc, + object: &str, + data: Vec, + token: &PoolMetaCasToken, + fence: &PoolMetaPersistenceFence<'_>, + phase: &'static str, + transaction_arm: &mut PoolMetaTransactionArm, +) -> Result +where + S: EcstoreObjectIO, +{ + let opts = pool_meta_cas_options(object, token, fence)?; + // Cancellation can happen at the very first poll of the storage future. + // Arm before dispatch, but not during read/encode/fence preflight. + let previous_phase = transaction_arm.phase; + transaction_arm.phase = Some(phase); + let outcome = execute_pool_meta_object_cas(pool, object, data, opts, fence, phase).await; + if !outcome.may_have_mutated { + transaction_arm.phase = previous_phase; + } + if let Some(source) = outcome.source + && (!matches!(source.as_ref(), Error::PreconditionFailed) + || transaction_arm + .source + .as_ref() + .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed))) + { + transaction_arm.source = Some(source); + } + outcome.result +} + +async fn save_pool_meta_phase( + pools: &[Arc], + data: &[u8], + tokens: &[PoolMetaCasToken], + fence: &PoolMetaPersistenceFence<'_>, + phase: &'static str, + transaction_arm: &mut PoolMetaTransactionArm, + #[cfg(feature = "e2e-test-hooks")] barrier: Option<&PoolMetaPhaseBarrier>, +) -> Vec> +where + S: EcstoreObjectIO, +{ + if pools.len() != tokens.len() { + return vec![Err(Error::other("pool metadata phase has inconsistent replica revisions"))]; + } + let options = tokens + .iter() + .map(|token| pool_meta_cas_options(POOL_META_NAME, token, fence)) + .collect::>(); + let previous_phase = transaction_arm.phase; + if options.iter().any(Result::is_ok) { + transaction_arm.phase = Some(phase); + } + // The caller owns one arm and the save/namespace guards for the whole phase. + // No replica may restore that arm while another write is still in flight. + let writes = futures::stream::iter(pools.iter().cloned().zip(options).enumerate().map( + |(pool_index, (pool, opts))| async move { + #[cfg(feature = "e2e-test-hooks")] + if opts.is_ok() + && let Some(barrier) = barrier + && let Err(err) = barrier.before_write(phase, pool_index, data).await + { + return (pool_index, PoolMetaCasWriteOutcome::not_dispatched(err)); + } + let outcome = match opts { + Ok(opts) => execute_pool_meta_object_cas(pool, POOL_META_NAME, data.to_vec(), opts, fence, phase).await, + Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err), + }; + #[cfg(feature = "e2e-test-hooks")] + let outcome = { + let mut outcome = outcome; + if let Some(barrier) = barrier + && let Err(err) = barrier.after_write(phase, pool_index, data, &outcome).await + && outcome.result.is_ok() + { + outcome.result = Err(err); + } + outcome + }; + (pool_index, outcome) + }, + )) + .buffer_unordered(4); + futures::pin_mut!(writes); + let mut results = Vec::with_capacity(pools.len()); + let mut may_have_mutated = false; + let mut source_pool_index: Option<(bool, usize)> = None; + while let Some((pool_index, outcome)) = writes.next().await { + may_have_mutated |= outcome.may_have_mutated; + if let Some(source) = outcome.source + && (outcome.may_have_mutated + || transaction_arm + .source + .as_ref() + .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed))) + && source_pool_index.is_none_or(|(previous_mutated, previous_index)| { + (outcome.may_have_mutated && !previous_mutated) + || (outcome.may_have_mutated == previous_mutated && pool_index < previous_index) + }) + { + // A later CAS rejection must not erase an I/O failure if draining is cancelled. + transaction_arm.source = Some(source); + source_pool_index = Some((outcome.may_have_mutated, pool_index)); + } + results.push((pool_index, outcome.result)); + } + if !may_have_mutated { + transaction_arm.phase = previous_phase; + } + results.sort_unstable_by_key(|(pool_index, _)| *pool_index); + results.into_iter().map(|(_, result)| result).collect() } /// Which pool replicas a cluster-identity write may touch. @@ -7724,48 +8109,116 @@ impl PoolMeta { }; let pending = encode_pool_meta_v3_envelope(&committed, revision, false, Some(&previous))?; let durable = encode_pool_meta_v3_envelope(&committed, revision, true, None)?; + let concurrent_phases = pools.len() > 1 + && selection.revision.is_generation_protocol() + && !selection.replica_state.needs_repair + && !bootstrap_generation_required + && matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_))); + #[cfg(feature = "e2e-test-hooks")] + let phase_barrier = if concurrent_phases && pools.len() == 4 { + PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await? + } else { + None + }; let mut pending_tokens = Vec::with_capacity(pools.len()); - for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) { - let object_info = - save_pool_meta_object_cas(pool, POOL_META_NAME, pending.clone(), token, fence, "prepare_cas", transaction_arm) - .await?; - let etag = object_info - .etag - .filter(|etag| !etag.trim().is_empty()) - .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?; - pending_tokens.push(PoolMetaCasToken::Existing(etag)); - } - - let mut commit_error = None; - let mut commit_succeeded = false; - #[cfg(test)] - let mut first_pool = true; - for (pool, token) in pools.iter().cloned().zip(&pending_tokens) { - match save_pool_meta_object_cas( - pool.clone(), - POOL_META_NAME, - durable.clone(), - token, + if concurrent_phases { + // Drain the entire prepare phase before validating ETags or returning an error. + for result in save_pool_meta_phase( + pools.as_slice(), + &pending, + &selection.cas_tokens, fence, - "commit_cas", + "prepare_cas", transaction_arm, + #[cfg(feature = "e2e-test-hooks")] + phase_barrier.as_ref(), ) .await { - Ok(_) => { - commit_succeeded = true; - #[cfg(test)] - if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence { - pause_pool_activation_after_durable_save(&pool, activation_fence).await; + let etag = result? + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?; + pending_tokens.push(PoolMetaCasToken::Existing(etag)); + } + } else { + for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) { + let object_info = save_pool_meta_object_cas( + pool, + POOL_META_NAME, + pending.clone(), + token, + fence, + "prepare_cas", + transaction_arm, + ) + .await?; + let etag = object_info + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?; + pending_tokens.push(PoolMetaCasToken::Existing(etag)); + } + } + + #[cfg(feature = "e2e-test-hooks")] + if let Some(barrier) = &phase_barrier + && barrier.arm.case == PoolMetaPhaseBarrierCase::PreparedAll + { + barrier.ready().await?; + } + let mut commit_error = None; + let mut commit_succeeded = false; + if concurrent_phases { + for result in save_pool_meta_phase( + pools.as_slice(), + &durable, + &pending_tokens, + fence, + "commit_cas", + transaction_arm, + #[cfg(feature = "e2e-test-hooks")] + phase_barrier.as_ref(), + ) + .await + { + match result { + Ok(_) => commit_succeeded = true, + Err(err) => { + commit_error.get_or_insert(err); } } - Err(err) => { - commit_error.get_or_insert(err); - } } + } else { #[cfg(test)] - { - first_pool = false; + let mut first_pool = true; + for (pool, token) in pools.iter().cloned().zip(&pending_tokens) { + match save_pool_meta_object_cas( + pool.clone(), + POOL_META_NAME, + durable.clone(), + token, + fence, + "commit_cas", + transaction_arm, + ) + .await + { + Ok(_) => { + commit_succeeded = true; + #[cfg(test)] + if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence { + pause_pool_activation_after_durable_save(&pool, activation_fence).await; + } + } + Err(err) => { + commit_error.get_or_insert(err); + } + } + #[cfg(test)] + { + first_pool = false; + } } } let confirmed = if fence.is_activation() { @@ -7784,6 +8237,12 @@ impl PoolMeta { "generation": confirmed.revision.generation, "transaction_id": confirmed.revision.transaction_id, })); + #[cfg(feature = "e2e-test-hooks")] + if let Some(barrier) = &phase_barrier + && barrier.arm.case == PoolMetaPhaseBarrierCase::BeforePublish + { + barrier.ready().await?; + } return Ok(confirmed.meta); } if !commit_succeeded { @@ -9079,7 +9538,7 @@ pub(crate) struct DecommissionPoolCapacityInfo { } impl DecommissionPoolCapacityInfo { - #[cfg(test)] + #[cfg(any(test, feature = "test-util"))] pub(crate) fn for_test( pool_index: usize, layout: DecommissionErasureLayout, @@ -9102,17 +9561,17 @@ impl DecommissionPoolCapacityInfo { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] type DecommissionCapacityInfoOverrides = std::sync::Mutex>>>; -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock = std::sync::OnceLock::new(); /// Queues capacity snapshots consumed in order by `get_decommission_all_pool_capacity_infos`; /// the final snapshot is retained and replayed for every subsequent sample, so tests never /// fall back to the host's real disk statistics once an override is installed. -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] pub(crate) fn set_decommission_capacity_info_overrides_for_test( store_id: uuid::Uuid, snapshots: Vec>, @@ -9124,7 +9583,7 @@ pub(crate) fn set_decommission_capacity_info_overrides_for_test( .insert(store_id, snapshots.into()); } -#[cfg(test)] +#[cfg(any(test, feature = "test-util"))] fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option> { let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES .get_or_init(|| std::sync::Mutex::new(HashMap::new())) @@ -9166,6 +9625,10 @@ struct DecommissionCapacityLockOrderBarrierState { cancel_before_start_entered: tokio::sync::Notify, cancel_before_start_release: tokio::sync::Notify, cancel_before_start_paused: AtomicBool, + cancel_target_timeout_entered: tokio::sync::Notify, + cancel_target_timeout_release: tokio::sync::Notify, + cancel_target_timeout_paused: AtomicBool, + cancel_target_timeouts: AtomicUsize, } #[cfg(test)] @@ -9205,6 +9668,10 @@ impl DecommissionCapacityLockOrderBarrier { cancel_before_start_entered: tokio::sync::Notify::new(), cancel_before_start_release: tokio::sync::Notify::new(), cancel_before_start_paused: AtomicBool::new(false), + cancel_target_timeout_entered: tokio::sync::Notify::new(), + cancel_target_timeout_release: tokio::sync::Notify::new(), + cancel_target_timeout_paused: AtomicBool::new(false), + cancel_target_timeouts: AtomicUsize::new(0), }); let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -9326,6 +9793,21 @@ impl DecommissionCapacityLockOrderBarrier { self.state.cancel_before_start_release.notify_one(); } + fn pause_cancel_target_timeout(&self) { + self.state.cancel_target_timeout_paused.store(true, Ordering::Release); + } + + async fn wait_until_cancel_target_timeout(&self) { + tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified()) + .await + .expect("cancel should observe contention on its target capacity fence"); + } + + fn release_cancel_target_timeout(&self) { + self.state.cancel_target_timeout_paused.store(false, Ordering::Release); + self.state.cancel_target_timeout_release.notify_one(); + } + #[cfg(feature = "test-util")] pub(crate) fn release_owner(&self) { self.state.owner_release.notify_one(); @@ -9368,6 +9850,7 @@ impl Drop for DecommissionCapacityLockOrderBarrier { self.state.external_object_capacity_probe_release.notify_one(); self.state.external_object_commit_phase_release.notify_one(); self.state.cancel_before_start_release.notify_one(); + self.state.cancel_target_timeout_release.notify_one(); let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) .lock() @@ -9426,6 +9909,24 @@ async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) { } } +#[cfg(test)] +async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) { + let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission capacity lock-order barrier should not be poisoned") + .as_ref() + .filter(|state| state.owner_store_id == store_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel); + barrier.cancel_target_timeout_entered.notify_one(); + if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) { + barrier.cancel_target_timeout_release.notified().await; + } + } +} + #[cfg(test)] fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) { let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER @@ -10063,6 +10564,7 @@ impl ECStore { async fn acquire_decommission_capacity_terminal_guards( &self, + source_pool_index: usize, plan: Option<&DecommissionCapacityTerminalFencePlan>, ) -> Result> { let Some(plan) = plan else { @@ -10086,26 +10588,69 @@ impl ECStore { "no storage pools available".to_string(), ) })?; - let mut guards = Vec::with_capacity(plan.target_pool_indices.len()); - for &target_pool_index in &plan.target_pool_indices { - let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"); - let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; - let guard = target_lock - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(|err| match err { - rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable { - mode: "write", - bucket: RUSTFS_META_BUCKET.to_string(), - object, - required, - achieved, - }, - other => Error::Lock(other), - })?; - guards.push(guard); + // Retry only target acquisition, never persistence. Keep the original + // owner/cohort pinned so a remote Clear/start cannot retarget a cancel. + let mut attempt = 1; + 'acquire_targets: loop { + let mut guards = Vec::with_capacity(plan.target_pool_indices.len()); + for &target_pool_index in &plan.target_pool_indices { + let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}"); + let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; + let started = std::time::Instant::now(); + match target_lock.get_write_lock(get_lock_acquire_timeout()).await { + Ok(guard) => guards.push(guard), + Err(err @ rustfs_lock::LockError::Timeout { .. }) => { + // Release the entire partial cohort before backoff or + // metadata reads; workers need these gates to settle I/O. + drop(guards); + #[cfg(test)] + pause_decommission_cancel_target_timeout(self.id).await; + if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS { + return Err(Error::Lock(err)); + } + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + state = "cancel_target_fence_retry", + pool_index = source_pool_index, + target_pool_index, + operation_id = %plan.operation_id, + generation = plan.generation, + owner_nonce = %plan.owner_nonce, + attempt, + max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS, + wait_ms = %started.elapsed().as_millis(), + error = %err, + "Decommission cancel will retry target capacity fencing" + ); + tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await; + let save_guard = self.pool_meta_save_gate.lock().await; + let (_read_guard, snapshot) = self + .acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed") + .await?; + if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) { + return Err(decommission_capacity_blocked_error( + "decommission capacity owner or target cohort changed while retrying terminal fences", + )); + } + attempt += 1; + continue 'acquire_targets; + } + Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => { + return Err(Error::NamespaceLockQuorumUnavailable { + mode: "write", + bucket: RUSTFS_META_BUCKET.to_string(), + object, + required, + achieved, + }); + } + Err(err) => return Err(Error::Lock(err)), + } + } + return Ok(guards); } - Ok(guards) } pub(crate) async fn acquire_external_decommission_capacity_fence( @@ -11847,7 +12392,7 @@ impl ECStore { } async fn get_decommission_all_pool_capacity_infos(&self) -> Result> { - #[cfg(test)] + #[cfg(any(test, feature = "test-util"))] if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) { return Ok(capacity_infos); } @@ -11900,6 +12445,25 @@ impl ECStore { })) } + pub(crate) async fn is_decommission_capacity_target_reserved( + &self, + owner: DecommissionCapacityOwner, + target_pool_index: usize, + ) -> Result { + let pool_meta = self.pool_meta.read().await; + let reservation = pool_meta + .pools + .get(owner.source_pool_index) + .and_then(|pool| pool.decommission.as_ref()) + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc())) + .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?; + Ok(reservation + .targets + .iter() + .any(|target| target.pool_index == target_pool_index)) + } + pub(crate) async fn select_decommission_capacity_target_pool( &self, owner: DecommissionCapacityOwner, @@ -12016,7 +12580,7 @@ impl ECStore { None }; let _capacity_target_guards = if acquire_runtime_fence { - self.acquire_decommission_capacity_terminal_guards(terminal_fence_plan.as_ref()) + self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref()) .await? } else { Vec::new() @@ -12703,6 +13267,9 @@ impl ECStore { self: &Arc, rx: CancellationToken, ) -> Result<()> { + #[cfg(test)] + let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints()); + #[cfg(not(test))] let endpoints = self.endpoints(); let index_cancelers = self.reserve_missing_local_decommission_routines(&rx, &endpoints).await?; if index_cancelers.is_empty() { @@ -13158,6 +13725,287 @@ impl ECStore { install_decommission_capacity_target_permit(self.id, target_pool_index, owner, target_guard).map(Some) } + #[cfg(feature = "test-util")] + pub async fn prepare_scanner_pause_backlog_retirement_for_test( + &self, + source_pool_index: usize, + source_bytes: usize, + ) -> Result<()> { + let source_bytes = source_bytes.max(1); + let total = source_bytes.saturating_mul(8).saturating_add(64 * 1024); + let capacities = (0..self.pools.len()) + .map(|pool_index| { + DecommissionPoolCapacityInfo::for_test( + pool_index, + DecommissionErasureLayout { data: 1, parity: 0 }, + if pool_index == source_pool_index { 0 } else { total }, + total, + if pool_index == source_pool_index { source_bytes } else { 0 }, + ) + }) + .collect(); + set_decommission_capacity_info_overrides_for_test(self.id, vec![capacities]); + self.save_current_pool_meta_for_decommission_start(&[source_pool_index], Vec::new()) + .await + .map(|_| ()) + } + + #[cfg(feature = "test-util")] + pub async fn retire_scanner_pause_backlog_for_test( + self: &Arc, + source_pool_index: usize, + source_set_index: usize, + ) -> Result<()> { + let set = self + .pools + .get(source_pool_index) + .and_then(|pool| pool.disk_set.get(source_set_index)) + .cloned() + .ok_or_else(|| Error::other("scanner retirement test requested an unknown set"))?; + let generation = self.active_decommission_generation(source_pool_index).await?; + let owner = self + .decommission_capacity_owner_for_worker(source_pool_index, generation) + .await?; + let expected = set + .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH) + .await? + .unwrap_or_default(); + match self + .retire_scanner_pause_backlog_entry(CancellationToken::new(), source_pool_index, generation, set, expected, owner) + .await? + { + DecommissionEntryAttemptOutcome::Complete => Ok(()), + DecommissionEntryAttemptOutcome::SourceChanged => Err(Error::other("scanner retirement source changed")), + } + } + + #[cfg(feature = "test-util")] + pub async fn stage_scanner_pause_backlog_retirement_intent_for_test( + &self, + source_pool_index: usize, + source_set_index: usize, + ) -> Result<()> { + let generation = self.active_decommission_generation(source_pool_index).await?; + let owner = self + .decommission_capacity_owner_for_worker(source_pool_index, generation) + .await? + .ok_or_else(|| Error::other("scanner retirement test has no capacity owner"))?; + let versions = self.pools[source_pool_index].disk_set[source_set_index] + .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH) + .await? + .ok_or_else(|| Error::other("scanner retirement test source is missing"))?; + let version = versions + .versions + .first() + .ok_or_else(|| Error::other("scanner retirement test source is empty"))?; + let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version)); + let size = usize::try_from(version.size).map_err(|_| Error::other("scanner retirement test source size is invalid"))?; + let target = self.select_decommission_capacity_target_pool(owner, size).await?; + let failed: Result<()> = self + .run_decommission_capacity_admitted_mutation(target, Some(owner), Some(size), || async { + Err(Error::other("injected scanner retirement target failure")) + }) + .await; + match failed { + Err(err) if err.to_string().contains("injected scanner retirement target failure") => Ok(()), + Err(err) => Err(err), + Ok(()) => Err(Error::other("scanner retirement test did not retain its target intent")), + } + } + + #[allow(clippy::too_many_arguments)] + async fn retire_scanner_pause_backlog_entry( + self: &Arc, + rx: CancellationToken, + idx: usize, + generation: OffsetDateTime, + set: Arc, + expected: FileInfoVersions, + capacity_owner: Option, + ) -> Result { + let store = Arc::clone(self); + // The disk layer may own a rename after its waiter is canceled. Keep + // the topology and object fences in that operation's owning task. + tokio::spawn(async move { + let operation_gate = store.ctx.data_movement_operation_gate(); + store + .run_guarded_decommission_side_effect(&rx, &operation_gate, || { + store.retire_scanner_pause_backlog_entry_inner(idx, generation, set, &expected, capacity_owner) + }) + .await + }) + .await + .map_err(Error::from)? + } + + async fn retire_scanner_pause_backlog_entry_inner( + &self, + idx: usize, + generation: OffsetDateTime, + set: Arc, + expected: &FileInfoVersions, + capacity_owner: Option, + ) -> Result { + use data_movement::scanner_backlog::{ + SCANNER_PAUSE_BACKLOG_PATH, persist_native_scanner_pause_backlog_replica, plan_scanner_pause_backlog_retirement, + read_scanner_pause_backlog_retirement_replicas, + }; + + if expected.versions.is_empty() && expected.free_versions.is_empty() { + // A canceled entry waiter may resume after the owned cleanup + // finished deleting its source. There is no remaining record to move. + return Ok(DecommissionEntryAttemptOutcome::Complete); + } + let [version] = expected.versions.as_slice() else { + return Err(Error::other("scanner pause backlog retirement requires exactly one source version")); + }; + if version.version_id.is_some_and(|version| !version.is_nil()) + || version.deleted + || version.tier_free_version() + || version.is_remote() + || !expected.free_versions.is_empty() + { + return Err(Error::other( + "scanner pause backlog retirement requires an unversioned local source record", + )); + } + let object_fence = self + .acquire_decommission_source_cleanup_fence(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, set.as_ref()) + .await?; + // Native replica writers acquire this same fixed object domain before + // durable pool metadata. Keep both fences through physical source deletion. + let save_guard = self.pool_meta_save_gate.lock().await; + let (pool_meta_guard, snapshot) = self + .acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed") + .await?; + ensure_decommission_generation(&snapshot, idx, generation)?; + let owner = capacity_owner.ok_or_else(|| { + decommission_capacity_blocked_error("scanner pause backlog retirement has no active capacity owner") + })?; + let reservation = snapshot.pools[idx] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc())) + .ok_or_else(|| decommission_capacity_blocked_error("scanner pause backlog retirement reservation is stale"))?; + let mutation_id = decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version); + if reservation.targets.iter().any(|target| { + target.pending_mutation_id == Some(mutation_id) + || target + .temporary_mutations + .iter() + .any(|mutation| mutation.mutation_id == mutation_id) + }) { + return Err(decommission_capacity_blocked_error( + "scanner pause backlog retirement has an unresolved target capacity intent", + )); + } + if snapshot.scanner_pause_backlog_pool_writable(idx) { + return Err(Error::other("scanner pause backlog source still belongs to writable membership")); + } + let sets: Vec<_> = self + .pools + .iter() + .enumerate() + .filter(|(pool_index, _)| *pool_index == idx || snapshot.scanner_pause_backlog_pool_writable(*pool_index)) + .flat_map(|(_, pool)| pool.disk_set.iter().cloned()) + .collect(); + let mut replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?; + if let Some(plan) = plan_scanner_pause_backlog_retirement(idx, &replicas)? { + let expected_stable_record = plan.stable_record.clone(); + let mut phases = Vec::with_capacity(3); + if let Some(record) = plan.seed_record { + phases.push(("seed", record)); + } + phases.push(("commit", plan.commit_record)); + phases.push(("stabilize", plan.stable_record)); + for (phase, record) in phases { + if object_fence.is_lock_lost() + || pool_meta_guard.is_lock_lost() + || !reservation.admits_owner(owner, OffsetDateTime::now_utc()) + { + return Err(decommission_capacity_blocked_error( + "scanner pause backlog retirement fence expired during native repair", + )); + } + for read in replicas.iter().filter(|read| read.replica.pool_index != idx) { + ensure_external_decommission_target_admission( + &snapshot, + read.replica.pool_index, + DecommissionCapacityAdmission::ScannerBacklog, + )?; + } + let results = join_all(replicas.iter().filter(|read| read.replica.pool_index != idx).map(|read| { + let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]); + let record = record.clone(); + let object_fence = &object_fence; + let pool_meta_guard = &pool_meta_guard; + async move { + let mut opts = ObjectOptions { + no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await, + ..Default::default() + }; + object_fence.add_namespace_lock_fence(&mut opts); + opts.add_namespace_lock_guard(pool_meta_guard); + persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase).await + } + })) + .await; + // Every dispatched native CAS has finished before an error can + // release the owned task's object and durable membership fences. + for result in results { + result?; + } + replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?; + if replicas + .iter() + .filter(|read| read.replica.pool_index != idx) + .any(|read| read.replica.data.as_deref() != Some(record.as_slice())) + { + return Err(Error::other( + "scanner pause backlog native repair did not persist every surviving replica", + )); + } + if let Some(next) = plan_scanner_pause_backlog_retirement(idx, &replicas)? + && next.stable_record != expected_stable_record + { + return Err(Error::other("scanner pause backlog native authority changed during repair")); + } + } + if plan_scanner_pause_backlog_retirement(idx, &replicas)?.is_some() { + return Err(Error::other("scanner pause backlog native repair is not fully committed and stable")); + } + } + if object_fence.is_lock_lost() + || pool_meta_guard.is_lock_lost() + || !reservation.admits_owner(owner, OffsetDateTime::now_utc()) + { + return Err(decommission_capacity_blocked_error( + "scanner pause backlog retirement fence expired before cleanup", + )); + } + let result = data_movement::cleanup_source_entry_if_unchanged( + set, + RUSTFS_META_BUCKET, + SCANNER_PAUSE_BACKLOG_PATH, + expected, + &[], + data_movement::SourceCleanupBucketFence { + expected_incarnation_id: None, + lifecycle_guard: None, + namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(), + object_mutation_fence: Some(&object_fence), + }, + "scanner pause backlog retirement", + ) + .await; + match result { + Ok(_) => Ok(DecommissionEntryAttemptOutcome::Complete), + Err(data_movement::SourceCleanupError::SourceChanged) => Ok(DecommissionEntryAttemptOutcome::SourceChanged), + Err(data_movement::SourceCleanupError::Storage(err)) => Err(err), + } + } + #[allow(clippy::too_many_arguments)] #[tracing::instrument(skip( self, @@ -13294,6 +14142,12 @@ impl ECStore { ); return Ok(DecommissionEntryAttemptOutcome::Complete); } + // Scanner caches describe their own erasure set and are rebuilt there. + // Copying one onto another set can overwrite unrelated cache contents or + // leave an unresolvable target-capacity intent after a conditional PUT. + if is_decommission_set_local_usage_cache(&bucket, &entry.name) { + return Ok(DecommissionEntryAttemptOutcome::Complete); + } let durable_ilm_record = if bucket == RUSTFS_META_BUCKET { classify_durable_ilm_record(&entry.name) .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))? @@ -13316,6 +14170,32 @@ impl ECStore { let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?; + if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) { + let outcome = self + .retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner) + .await?; + if matches!(outcome, DecommissionEntryAttemptOutcome::Complete) { + let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; + if let Some(version) = fivs.versions.first() + && counted_versions.insert((version.version_id, false)) + { + count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?; + } + track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?; + drop(pool_meta); + self.track_decommission_entry_progress_stage( + idx, + generation, + &bucket, + &entry.name, + DECOMMISSION_STAGE_ENTRY_FINISHED, + ) + .await?; + } + return Ok(outcome); + } + let pending_mutations = if let Some(owner) = capacity_owner { self.pool_meta .read() @@ -13851,7 +14731,7 @@ impl ECStore { let bucket = bucket.clone(); - let rd = match set + let read_result = set .get_object_reader( bucket.as_str(), &encode_dir_object(&version.name), @@ -13859,8 +14739,11 @@ impl ECStore { HeaderMap::new(), &decommission_object_migration_read_opts(version_id.clone()), ) - .await - { + .await; + #[cfg(test)] + let read_result = + decommission_test_wrap_result("object_read", &bucket, &version.name, version_attempt, read_result); + let rd = match read_result { Ok(rd) => rd, Err(err) => { if is_err_object_not_found(&err) || is_err_version_not_found(&err) { @@ -13869,15 +14752,6 @@ impl ECStore { break; } - if !ignore { - // - if bucket == RUSTFS_META_BUCKET && version.name.contains(DATA_USAGE_CACHE_NAME) { - ignore = true; - error!("decommission_pool: ignore data usage cache {}", &version.name); - break; - } - } - failure = true; if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS { error!( @@ -16845,7 +17719,7 @@ impl ECStore { return; } - if bucket_name == RUSTFS_META_BUCKET && entry.name.contains(DATA_USAGE_CACHE_NAME) { + if is_decommission_set_local_usage_cache(&bucket_name, &entry.name) { return; } @@ -17163,6 +18037,361 @@ mod tests { use crate::storage_api_contracts::multipart::MultipartOperations as _; use serde::Serialize; + #[test] + fn decommission_set_local_usage_cache_classification_is_exact() { + for object in [ + "buckets/.usage-cache.bin", + "buckets/.usage-cache.bin.bkp", + "buckets/photos/.usage-cache.bin", + "buckets/photos/.usage-cache.bin.bkp", + ] { + assert!(is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}"); + assert!(!is_decommission_set_local_usage_cache("user-bucket", object), "{object}"); + } + for object in [ + "buckets/.usage.v2.json", + "buckets/.usage.v2.json.bkp", + "buckets/.usage-cache.bin.extra", + "buckets/.usage-cache.bin.bkp.extra", + "buckets/prefix.usage-cache.bin", + "buckets/photos/.usage-cache.bin.bkp.bkp", + "buckets/photos/nested/.usage-cache.bin", + "buckets//.usage-cache.bin", + "buckets/../.usage-cache.bin", + "buckets/./.usage-cache.bin", + "buckets/.usage-cache.bin/child", + "config/.usage-cache.bin", + "buckets-other/.usage-cache.bin", + ".usage-cache.bin", + ] { + assert!(!is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}"); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_keeps_set_local_usage_caches_out_of_target_capacity() { + use crate::object_api::PutObjReader; + use tokio::io::AsyncReadExt as _; + + // Keep the scenario's large setup and migration futures off the test + // future so ordinary metadata I/O retains the default thread stack. + let (_temp_dirs, store, _other_store) = + Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await; + let user_bucket = "decommission-usage-cache-control"; + Box::pin(store.make_bucket(user_bucket, &MakeBucketOptions::default())) + .await + .expect("create the ordinary-object control bucket"); + let incarnation = Box::pin(store.bucket_incarnation_id(user_bucket)) + .await + .expect("control bucket incarnation"); + let source_time = OffsetDateTime::now_utc(); + let cache_objects = [ + "buckets/.usage-cache.bin", + "buckets/.usage-cache.bin.bkp", + "buckets/photos/.usage-cache.bin", + "buckets/photos/.usage-cache.bin.bkp", + ]; + let conflict_object = "buckets/.usage-cache.bin.conflict"; + let source_read_failure_object = "buckets/.usage-cache.bin.read-error"; + let source_body = b"source set cache"; + let target_body = b"independent older target set cache"; + for object in cache_objects.into_iter().chain([conflict_object]) { + for (pool_index, body, mod_time) in [ + (0, source_body.as_slice(), source_time), + (1, target_body.as_slice(), source_time - Duration::seconds(1)), + ] { + // Scanner cache persistence writes directly to its own set. + store.pools[pool_index] + .get_disks_by_key(object) + .put_object( + RUSTFS_META_BUCKET, + object, + &mut PutObjReader::from_vec(body.to_vec()), + &ObjectOptions { + mod_time: Some(mod_time), + ..Default::default() + }, + ) + .await + .expect("seed distinct native set-local objects"); + } + } + let controls = [ + (RUSTFS_META_BUCKET, "buckets/.usage.v2.json"), + (RUSTFS_META_BUCKET, "buckets/photos/.usage-cache.bin.extra"), + (user_bucket, "ordinary-object"), + (user_bucket, "buckets/.usage-cache.bin"), + ]; + for (bucket, object) in controls { + store.pools[0] + .put_object( + bucket, + object, + &mut PutObjReader::from_vec(b"ordinary object contents".to_vec()), + &ObjectOptions { + expected_bucket_incarnation_id: (bucket == user_bucket).then_some(incarnation), + ..Default::default() + }, + ) + .await + .expect("seed a control that must migrate"); + } + store.pools[0] + .put_object( + RUSTFS_META_BUCKET, + source_read_failure_object, + &mut PutObjReader::from_vec(source_body.to_vec()), + &ObjectOptions::default(), + ) + .await + .expect("seed a similarly named object whose source read will fail"); + + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384), + DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0), + ]], + ); + Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new())) + .await + .expect("activate the decommission capacity reservation"); + + for object in cache_objects { + Box::pin(store.decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + RUSTFS_META_BUCKET.to_string(), + store.pools[0].get_disks_by_key(object), + )) + .await + .expect("set-local cache must not enter cross-pool migration"); + for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] { + let mut reader = store.pools[pool_index] + .get_disks_by_key(object) + .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("each set must retain its own cache"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read retained cache"); + assert_eq!(actual, expected, "pool {pool_index}, {object}"); + } + let meta = store.pool_meta.read().await; + let info = meta.pools[0].decommission.as_ref().expect("decommission progress"); + assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 0)); + assert_eq!((info.bytes_done, info.bytes_failed), (0, 0)); + let reservation = info.capacity_reservation.as_ref().expect("capacity reservation"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.consumed_target_physical_bytes, 0); + assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none())); + } + let mut persisted = PoolMeta::default(); + Box::pin(persisted.load_no_lock_from_replicas(store.pools.clone())) + .await + .expect("reload durable capacity intents after cache entries"); + let reservation = persisted.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("durable reservation"); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none())); + + let injected_reads = Arc::new(AtomicUsize::new(0)); + let observed_reads = Arc::clone(&injected_reads); + let read_fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _, success| { + if stage == "object_read" && bucket == RUSTFS_META_BUCKET && object == source_read_failure_object && success { + observed_reads.fetch_add(1, Ordering::SeqCst); + return true; + } + false + })); + Box::pin(store.decommission_entry_for_test( + 0, + MetaCacheEntry { + name: source_read_failure_object.to_string(), + ..Default::default() + }, + RUSTFS_META_BUCKET.to_string(), + store.pools[0].get_disks_by_key(source_read_failure_object), + )) + .await + .expect("entry must record the non-NotFound source read failure"); + drop(read_fault); + assert_eq!(injected_reads.load(Ordering::SeqCst), DECOMMISSION_VERSION_COPY_ATTEMPTS); + { + let meta = store.pool_meta.read().await; + let info = meta.pools[0].decommission.as_ref().expect("source read failure progress"); + assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 1)); + assert_eq!(info.bytes_failed, source_body.len()); + assert_eq!( + info.capacity_reservation + .as_ref() + .expect("reservation") + .pending_target_physical_bytes, + 0 + ); + } + let mut retained = store.pools[0] + .get_object_reader( + RUSTFS_META_BUCKET, + source_read_failure_object, + None, + HeaderMap::new(), + &ObjectOptions::default(), + ) + .await + .expect("source read failure must retain the source"); + let mut retained_body = Vec::new(); + retained + .stream + .read_to_end(&mut retained_body) + .await + .expect("read retained source"); + assert_eq!(retained_body, source_body); + drop(retained); + let target_err = store.pools[1] + .get_object_info(RUSTFS_META_BUCKET, source_read_failure_object, &ObjectOptions::default()) + .await + .expect_err("failed source read must not create a target object"); + assert!(is_err_object_not_found(&target_err), "unexpected target state: {target_err:?}"); + + for (bucket, object) in controls { + Box::pin(store.decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + bucket.to_string(), + store.pools[0].get_disks_by_key(object), + )) + .await + .expect("ordinary and similarly named objects must migrate"); + let mut reader = store.pools[1] + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("control must exist on the target"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read migrated control"); + assert_eq!(actual, b"ordinary object contents", "{bucket}/{object}"); + let err = store.pools[0] + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect_err("migrated control must be removed from the source"); + assert!(is_err_object_not_found(&err), "{bucket}/{object}: {err:?}"); + } + + Box::pin(store.decommission_entry_for_test( + 0, + MetaCacheEntry { + name: conflict_object.to_string(), + ..Default::default() + }, + RUSTFS_META_BUCKET.to_string(), + store.pools[0].get_disks_by_key(conflict_object), + )) + .await + .expect("entry must record a real conditional-copy failure"); + let meta = store.pool_meta.read().await; + let info = meta.pools[0].decommission.as_ref().expect("final progress"); + assert_eq!(info.items_decommissioned, controls.len()); + assert_eq!( + info.items_decommission_failed, 2, + "similar names must not hide read or migration failures" + ); + assert_eq!(info.bytes_failed, source_body.len() * 2); + drop(meta); + for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] { + let mut reader = store.pools[pool_index] + .get_object_reader(RUSTFS_META_BUCKET, conflict_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("failed migration must preserve both objects"); + let mut actual = Vec::new(); + reader.stream.read_to_end(&mut actual).await.expect("read conflict object"); + assert_eq!(actual, expected); + } + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_final_sweep_excludes_only_set_local_usage_caches() { + use crate::object_api::PutObjReader; + + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await; + for object in [ + "buckets/.usage-cache.bin", + "buckets/.usage-cache.bin.bkp", + "buckets/photos/.usage-cache.bin", + "buckets/photos/.usage-cache.bin.bkp", + ] { + store.pools[0] + .get_disks_by_key(object) + .put_object( + RUSTFS_META_BUCKET, + object, + &mut PutObjReader::from_vec(b"set-local cache".to_vec()), + &ObjectOptions::default(), + ) + .await + .expect("seed each supported set-local cache path"); + } + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384), + DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("activate the final-sweep generation"); + let generation = store.active_decommission_generation(0).await.expect("active generation"); + store + .check_after_decommission(0, &CancellationToken::new(), generation) + .await + .expect("the four set-local cache forms must not block the final sweep"); + + for object in [ + "buckets/.usage-cache.bin.extra", + "buckets/photos/.usage-cache.bin.bkp.extra", + "buckets/.usage.v2.json", + ] { + let source_set = store.pools[0].get_disks_by_key(object); + source_set + .put_object( + RUSTFS_META_BUCKET, + object, + &mut PutObjReader::from_vec(b"unmigrated ordinary metadata".to_vec()), + &ObjectOptions::default(), + ) + .await + .expect("seed ordinary metadata that must prevent completion"); + let err = store + .check_after_decommission(0, &CancellationToken::new(), generation) + .await + .expect_err("a remaining similar name or global usage snapshot must block completion"); + assert!(err.to_string().contains("after decommissioning"), "unexpected final-sweep error: {err:?}"); + assert!(err.to_string().contains(object), "the final sweep must identify {object}: {err:?}"); + source_set + .delete_object(RUSTFS_META_BUCKET, object, ObjectOptions::default()) + .await + .expect("remove only the ordinary-metadata control before the next sweep"); + } + store + .check_after_decommission(0, &CancellationToken::new(), generation) + .await + .expect("only the four set-local caches remain after removing the controls"); + } + #[test] fn pool_activation_fleet_proof_error_classifier_matches_only_retryable_proof_failures() { assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))); @@ -17594,6 +18823,429 @@ mod tests { assert_eq!(reservation.inflight_target_physical_bytes, 0); } + async fn start_target_fenced_cancel_test( + store: &Arc, + first_target_free: usize, + ) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) { + crate::services::rebalance::promote_test_pool_meta_to_v2(store).await; + let layout = DecommissionErasureLayout { data: 1, parity: 0 }; + set_decommission_capacity_info_overrides_for_test( + store.id, + vec![vec![ + DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10), + DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free), + DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0), + ]], + ); + store + .save_current_pool_meta_for_decommission_start(&[0], Vec::new()) + .await + .expect("persist an active target-fenced decommission"); + let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0) + .expect("active decommission should have a valid fence plan") + .expect("active decommission should retain its reservation"); + assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION); + let canceler = DecommissionCanceler::new(CancellationToken::new()); + *store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None]; + (plan, canceler) + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await; + assert_eq!(plan.target_pool_indices, vec![2]); + let target_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2")) + .await + .expect("create the active migration target gate"); + let target_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("hold the target gate across the first cancel acquisition timeout"); + let movement_gate = store.ctx.data_movement_operation_gate(); + let movement_guard = movement_gate.read().await; + let cancel_store = Arc::clone(&store); + let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await }); + + tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel) + .await + .expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains"); + assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing"); + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the active reservation must remain readable while cancel waits"); + assert_eq!( + decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"), + Some(plan), + "a failed target acquisition must not release or replace the durable reservation" + ); + assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled); + + drop(target_guard); + tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled()) + .await + .expect("cancel should persist and signal its worker after target contention is released"); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("reload the committed cancellation from native replicas"); + let info = durable.pools[0] + .decommission + .as_ref() + .expect("canceled state must remain durable"); + assert!(info.canceled); + assert!(!info.complete && !info.failed); + let reservation = info + .capacity_reservation + .as_ref() + .expect("terminal capacity accounting must remain inspectable"); + assert!(!reservation.active()); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel) + .await + .expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling"); + drop(movement_guard); + tokio::time::timeout(std::time::Duration::from_secs(30), cancel) + .await + .expect("cancel should return after in-flight movement quiesces") + .expect("cancel task should not panic") + .expect("the same cancel request should finish its durable terminal transition"); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await; + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + let target_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2")) + .await + .expect("create the persistently contended target gate"); + let target_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("hold the target gate for every cancel attempt"); + + let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0)) + .await + .expect("target contention must not retry indefinitely") + .expect_err("exhausting the acquisition budget must not report cancellation success"); + match err { + Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => { + assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest"); + assert_eq!(timeout, std::time::Duration::from_secs(1)); + } + other => panic!("expected the final typed target-lock timeout, got {other:?}"), + } + assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3); + assert!(!canceler.is_cancelled()); + assert!(canceler.is_active()); + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("reload the reservation after the canceled request exhausted its budget"); + assert_eq!( + decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"), + Some(plan) + ); + assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled); + drop(target_guard); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { + let (_temp_dirs, store, other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await; + assert_eq!(plan.target_pool_indices, vec![1, 2]); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + barrier.pause_cancel_target_timeout(); + let target_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2")) + .await + .expect("create the second target gate"); + let target_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("block cancellation after it acquires the first target"); + let cancel_store = Arc::clone(&store); + let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await }); + barrier.wait_until_cancel_target_timeout().await; + + let first_target_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1")) + .await + .expect("create the partial-cohort release probe"); + let first_target_guard = first_target_lock + .get_write_lock(std::time::Duration::from_secs(1)) + .await + .expect("cancel must release its first target before retrying the blocked second target"); + drop(first_target_guard); + drop(target_guard); + + other_store + .reload_pool_meta() + .await + .expect("load the active generation on the remote node"); + other_store + .decommission_cancel(0) + .await + .expect("the remote node should cancel the old generation"); + other_store + .clear_decommission(0) + .await + .expect("the remote node should clear the old generation"); + let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await; + assert_ne!(replacement_plan.operation_id, plan.operation_id); + + barrier.release_cancel_target_timeout(); + let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel) + .await + .expect("the stale request must finish without retrying a replacement operation") + .expect("the stale cancel task should not panic") + .expect_err("the original request must not cancel a remotely replaced generation"); + assert!(err.to_string().contains("changed while retrying terminal fences")); + assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1); + assert!(!replacement_canceler.is_cancelled()); + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("reload the replacement after the stale cancel returns"); + assert_eq!( + decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"), + Some(replacement_plan) + ); + assert!( + !durable.pools[0] + .decommission + .as_ref() + .expect("replacement decommission") + .canceled + ); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await; + let owner = DecommissionCapacityOwner { + source_pool_index: 0, + operation_id: plan.operation_id, + generation: plan.generation, + owner_nonce: plan.owner_nonce, + mutation_id: Some(uuid::Uuid::new_v4()), + }; + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let mutation_store = Arc::clone(&store); + let mutation = tokio::spawn(async move { + mutation_store + .run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async { + entered_tx + .send(()) + .expect("the test should observe the admitted target mutation"); + release_rx.await.expect("the test should release the in-flight mutation"); + Ok(()) + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx) + .await + .expect("target mutation should reach its controlled I/O phase") + .expect("target admission should succeed before cancel starts"); + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + let cancel_store = Arc::clone(&store); + let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await }); + barrier.wait_until_cancel_target_timeout().await; + assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal"); + cancel.abort(); + assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled()); + + release_tx + .send(()) + .expect("the mutation must still be alive after the caller disconnects"); + tokio::time::timeout(std::time::Duration::from_secs(30), mutation) + .await + .expect("the in-flight mutation must be able to settle without a metadata lock cycle") + .expect("the mutation task should not panic") + .expect("the admitted mutation must settle before the target gate is released"); + tokio::time::timeout(std::time::Duration::from_secs(30), async { + loop { + if store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .is_some_and(|info| info.canceled) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .expect("the detached cancel transaction must finish after target contention clears"); + assert!(canceler.is_cancelled()); + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("reload cancellation after the RPC waiter was dropped"); + let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission"); + assert!(info.canceled && !info.failed && !info.complete); + let reservation = info + .capacity_reservation + .as_ref() + .expect("inspect settled capacity accounting"); + assert!(!reservation.active()); + assert_eq!(reservation.pending_target_physical_bytes, 0); + assert_eq!(reservation.inflight_target_physical_bytes, 0); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await; + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + barrier.pause_cancel_target_timeout(); + let target_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2")) + .await + .expect("create the target gate preceding the failed save"); + let target_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("force one target acquisition retry before persistence"); + let save_calls = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&save_calls); + let cancel_store = Arc::clone(&store); + let owner = canceler.clone(); + let cancel = tokio::spawn(async move { + cancel_store + .decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move { + calls.fetch_add(1, Ordering::AcqRel); + Err(Error::Timeout) + }) + .await + }); + barrier.wait_until_cancel_target_timeout().await; + assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence"); + drop(target_guard); + barrier.release_cancel_target_timeout(); + let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel) + .await + .expect("a persistence failure must end the cancellation transaction") + .expect("the failed-save task should not panic") + .expect_err("the injected durable-save failure must reach the caller"); + assert!(matches!(err, Error::Timeout)); + assert_eq!(save_calls.load(Ordering::Acquire), 1); + assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1); + assert!(!canceler.is_cancelled()); + assert!(canceler.is_active()); + store + .ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes") + .await + .expect_err("a failed durable save must retain the existing recovery gate"); + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the original reservation should remain readable after the injected save failure"); + assert_eq!( + decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"), + Some(plan) + ); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn decommission_cancel_target_retry_converges_after_successive_contenders() { + temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async { + let (_temp_dirs, store, _other_store) = + crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await; + let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await; + let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id); + barrier.pause_cancel_target_timeout(); + let target_lock = store.pools[0] + .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2")) + .await + .expect("create the repeatedly contended target gate"); + let first_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("the first contender should own the target gate"); + let cancel_store = Arc::clone(&store); + let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await }); + barrier.wait_until_cancel_target_timeout().await; + assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1); + drop(first_guard); + let second_guard = target_lock + .get_write_lock(std::time::Duration::from_secs(30)) + .await + .expect("a second contender should be able to acquire between cancel attempts"); + barrier.release_cancel_target_timeout(); + barrier.pause_cancel_target_timeout(); + barrier.wait_until_cancel_target_timeout().await; + assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2); + assert!(!canceler.is_cancelled()); + drop(second_guard); + barrier.release_cancel_target_timeout(); + + tokio::time::timeout(std::time::Duration::from_secs(30), cancel) + .await + .expect("cancel should converge after the repeated contention ends") + .expect("the cancel task should not panic") + .expect("the third target acquisition should permit a durable cancellation"); + assert!(canceler.is_cancelled()); + let mut durable = PoolMeta::default(); + durable + .load_no_lock_from_replicas(store.pools.clone()) + .await + .expect("the successful retried cancellation must survive a native metadata reload"); + assert!( + durable.pools[0] + .decommission + .as_ref() + .expect("canceled decommission") + .canceled + ); + assert!( + decommission_capacity_terminal_fence_plan(&durable, 0) + .expect("valid terminal metadata") + .is_none() + ); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() { @@ -19874,6 +21526,55 @@ mod tests { assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown))); } + #[test] + fn decommission_target_gate_retry_recognizes_multipart_part_errors() { + let wrap = |inner: Error| { + data_movement::data_movement_part_stage_error_for_test( + "decommission_object", + "put_object_part", + "bucket-a", + "object-a", + 1, + inner, + ) + }; + let gate_busy_message = + format!("{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"); + let wrapped = wrap(decommission_capacity_blocked_error(&gate_busy_message)); + assert!(is_decommission_capacity_target_gate_busy(&wrapped)); + assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), Some(7)); + assert_eq!( + wrapped.to_string(), + format!( + "Io error: decommission_object: put_object_part failed for bucket-a/object-a part 1: {}", + decommission_capacity_blocked_error(&gate_busy_message) + ) + ); + + for unrelated in [ + Error::SlowDown, + Error::DiskFull, + decommission_capacity_blocked_error("target capacity is exhausted"), + Error::other(gate_busy_message), + ] { + let wrapped = wrap(unrelated); + assert!(!is_decommission_capacity_target_gate_busy(&wrapped)); + assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), None); + } + for missing_target in [ + Error::FileNotFound, + Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string()), + Error::VersionNotFound("bucket-a".to_string(), "object-a".to_string(), "version-a".to_string()), + ] { + assert!(is_decommission_copy_cleanup_safe_error(&missing_target)); + assert!( + !is_decommission_copy_cleanup_safe_error(&wrap(missing_target)), + "a missing target part must never authorize source cleanup" + ); + } + assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull))); + } + #[test] fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() { let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull)); @@ -21491,6 +23192,966 @@ mod pools_tests { } } + mod phase_tests { + use super::super as implementation; + use super::*; + + #[derive(Clone, Copy, Debug, Default)] + enum WriteFault { + #[default] + None, + Reject, + Timeout, + TimeoutAfterWrite, + EmptyEtag, + OmitWrite, + } + + #[derive(Clone, Debug, Default)] + struct WriteStep { + gate: Option>, + fault: WriteFault, + } + + #[derive(Clone, Debug)] + struct WriteCall { + pool: usize, + phase: usize, + if_match: Option, + } + + #[derive(Debug, Default)] + struct WriteTrace { + calls: StdMutex>, + active: AtomicUsize, + maximum: AtomicUsize, + finished: AtomicUsize, + changed: tokio::sync::Notify, + } + + impl WriteTrace { + async fn wait_for(&self, calls: usize, finished: usize) { + tokio::time::timeout(StdDuration::from_secs(3), async { + loop { + let changed = self.changed.notified(); + if self.calls.lock().expect("trace lock").len() >= calls + && self.finished.load(Ordering::SeqCst) >= finished + { + return; + } + changed.await; + } + }) + .await + .expect("phase must reach the requested barrier"); + } + + fn commit_count(&self) -> usize { + self.calls + .lock() + .expect("trace lock") + .iter() + .filter(|call| call.phase == 1) + .count() + } + } + + struct ActiveWrite<'a>(&'a WriteTrace); + impl Drop for ActiveWrite<'_> { + fn drop(&mut self) { + self.0.active.fetch_sub(1, Ordering::SeqCst); + self.0.finished.fetch_add(1, Ordering::SeqCst); + self.0.changed.notify_one(); + } + } + + #[derive(Debug)] + struct PhaseStorage { + inner: PartialPoolMetaWriteStorage, + pool: usize, + steps: [WriteStep; 2], + trace: Arc, + } + + #[async_trait::async_trait] + impl ObjectIO for PhaseStorage { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = crate::object_api::ObjectOptions; + type ObjectInfo = crate::object_api::ObjectInfo; + type GetObjectReader = crate::object_api::GetObjectReader; + type PutObjectReader = crate::object_api::PutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + range: Option, + headers: Self::HeaderMap, + opts: &Self::ObjectOptions, + ) -> Result { + self.inner.get_object_reader(bucket, object, range, headers, opts).await + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> Result { + if object != POOL_META_NAME { + return self.inner.put_object(bucket, object, data, opts).await; + } + let mut payload = Vec::new(); + data.stream.read_to_end(&mut payload).await?; + let phase = match implementation::decode_pool_meta_replica(payload.clone()) { + implementation::PoolMetaReplica::Valid { committed, .. } => usize::from(committed), + _ => panic!("phase fixture must receive valid pool metadata"), + }; + assert!(opts.max_parity && opts.no_lock); + assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrained); + self.trace.calls.lock().expect("trace lock").push(WriteCall { + pool: self.pool, + phase, + if_match: opts + .http_preconditions + .as_ref() + .and_then(HTTPPreconditions::if_match_value) + .map(str::to_owned), + }); + let active = self.trace.active.fetch_add(1, Ordering::SeqCst) + 1; + self.trace.maximum.fetch_max(active, Ordering::SeqCst); + let _active = ActiveWrite(&self.trace); + self.trace.changed.notify_one(); + let step = &self.steps[phase]; + if let Some(gate) = &step.gate { + gate.acquire().await.expect("phase gate stays open").forget(); + } + match step.fault { + WriteFault::Reject => return Err(Error::PreconditionFailed), + WriteFault::Timeout => return Err(Error::Timeout), + WriteFault::OmitWrite => { + return Ok(crate::object_api::ObjectInfo { + etag: Some("uncommitted".to_owned()), + ..Default::default() + }); + } + _ => {} + } + let mut data = crate::object_api::PutObjReader::from_vec(payload); + let mut result = self.inner.put_object(bucket, object, &mut data, opts).await?; + match step.fault { + WriteFault::TimeoutAfterWrite => return Err(Error::Timeout), + WriteFault::EmptyEtag => result.etag = Some(" ".to_owned()), + _ => {} + } + Ok(result) + } + } + + struct Fixture { + pools: Vec>, + trace: Arc, + previous: implementation::PoolMetaCommittedCandidate, + desired: PoolMeta, + pending: Vec, + durable: Vec, + revision: implementation::PoolMetaRevision, + cluster_id: uuid::Uuid, + } + + impl Fixture { + fn new(steps: Vec<[WriteStep; 2]>) -> Self { + let cluster_id = uuid::Uuid::new_v4(); + let meta = PoolMeta { + version: POOL_META_GENERATION_VERSION, + pools: (0..steps.len()) + .map(|index| { + decommission_test_pool_status( + index, + (index == 0).then(|| PoolDecommissionInfo { + start_time: Some(OffsetDateTime::UNIX_EPOCH), + start_size: 4096, + total_size: 16384, + current_size: 8192, + items_decommissioned: 7, + items_decommission_failed: 1, + bytes_done: 4096, + bytes_failed: 512, + queued_buckets: vec!["remaining-a".to_owned(), "remaining-b".to_owned()], + decommissioned_buckets: vec!["completed".to_owned()], + bucket: "remaining-a".to_owned(), + prefix: "objects/".to_owned(), + object: "objects/007".to_owned(), + ..Default::default() + }), + ) + }) + .collect(), + ..Default::default() + }; + let previous_revision = implementation::PoolMetaRevision { + version: POOL_META_GENERATION_VERSION, + cluster_id: Some(cluster_id), + epoch: 1, + generation: 7, + transaction_id: Some(uuid::Uuid::new_v4()), + }; + let previous = implementation::PoolMetaCommittedCandidate { + canonical: implementation::encode_pool_meta_v3_envelope(&meta, previous_revision, true, None) + .expect("committed predecessor"), + meta: meta.clone(), + revision: previous_revision, + }; + let mut desired = meta; + desired.pools[0].last_update += Duration::seconds(1); + let progress = desired.pools[0].decommission.as_mut().expect("active decommission"); + progress.items_decommissioned += 1; + progress.bytes_done += 512; + progress.current_size += 512; + progress.object = "objects/008".to_owned(); + let revision = implementation::PoolMetaRevision { + generation: 8, + transaction_id: Some(uuid::Uuid::new_v4()), + ..previous_revision + }; + let pending = implementation::encode_pool_meta_v3_envelope(&desired, revision, false, Some(&previous)) + .expect("pending candidate"); + let durable = + implementation::encode_pool_meta_v3_envelope(&desired, revision, true, None).expect("committed candidate"); + let identity = implementation::initialized_pool_meta_identity_for_test(cluster_id, 1).expect("identity"); + let trace = Arc::new(WriteTrace::default()); + let pools = steps + .into_iter() + .enumerate() + .map(|(index, steps)| { + Arc::new(PhaseStorage { + inner: PartialPoolMetaWriteStorage { + stored: StdMutex::new(Some((previous.canonical.clone(), format!("initial-{index}")))), + identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))), + revision: AtomicUsize::new(index * 10), + ..Default::default() + }, + pool: index, + steps, + trace: trace.clone(), + }) + }) + .collect(); + Self { + pools, + trace, + previous, + desired, + pending, + durable, + revision, + cluster_id, + } + } + + fn state(&self) -> implementation::PoolMetaWriteState { + implementation::PoolMetaWriteState::for_startup(self.cluster_id, false) + } + + fn tokens(&self) -> Vec { + self.pools + .iter() + .map(|pool| { + PoolMetaCasToken::Existing( + pool.inner + .stored + .lock() + .expect("fixture storage") + .as_ref() + .expect("existing replica") + .1 + .clone(), + ) + }) + .collect() + } + } + + fn steps(count: usize) -> Vec<[WriteStep; 2]> { + (0..count).map(|_| [WriteStep::default(), WriteStep::default()]).collect() + } + + fn signal() -> Option> { + Some(Arc::default()) + } + + async fn prepare( + fixture: &Fixture, + fence: &PoolMetaPersistenceFence<'_>, + arm: &mut implementation::PoolMetaTransactionArm, + ) -> Vec> { + implementation::save_pool_meta_phase( + &fixture.pools, + &fixture.pending, + &fixture.tokens(), + fence, + "prepare_cas", + arm, + #[cfg(feature = "e2e-test-hooks")] + None, + ) + .await + } + + #[tokio::test] + async fn pool_meta_v3_phase_concurrency_is_bounded() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(6); + for step in &mut config { + step[0].gate = Some(gate.clone()); + } + let fixture = Fixture::new(config); + let state = fixture.state(); + let mut arm = state.arm_transaction(); + let fence = PoolMetaPersistenceFence::Distributed(signal()); + let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm)); + tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("blocked writes finished") } + assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4); + gate.add_permits(1); + tokio::select! { _ = fixture.trace.wait_for(5, 1) => {}, _ = &mut pending => panic!("unreleased writes finished") } + assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 5); + gate.add_permits(5); + assert!(pending.await.into_iter().all(|result| result.is_ok())); + assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 4); + assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0); + arm.disarm(); + } + + #[tokio::test] + async fn pool_meta_v3_commit_waits_for_all_prepare_etags() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(4); + config[3][0].gate = Some(gate.clone()); + let fixture = Fixture::new(config); + let mut state = fixture.state(); + let mut save = Box::pin( + fixture + .desired + .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]), + ); + tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") } + assert_eq!(fixture.trace.commit_count(), 0); + gate.add_permits(1); + save.await.expect("all prepares and commits succeed").disarm(); + let calls = fixture.trace.calls.lock().expect("trace"); + for call in calls.iter().filter(|call| call.phase == 1) { + assert_eq!(call.if_match, Some(format!("pool-meta-test-{}", call.pool * 10 + 1))); + } + assert_eq!(calls.iter().filter(|call| call.phase == 1).count(), 4); + } + + #[tokio::test] + async fn pool_meta_v3_empty_prepare_etag_never_starts_commit() { + let mut config = steps(4); + config[3][0].fault = WriteFault::EmptyEtag; + let fixture = Fixture::new(config); + let mut state = fixture.state(); + let err = fixture + .desired + .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]) + .await + .expect_err("a missing conditional-write revision must fail"); + assert!(err.to_string().contains("without a conditional-write revision")); + assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4); + assert_eq!(fixture.trace.commit_count(), 0); + assert!(state.ensure_write_safe("empty etag wrote pending data").is_err()); + } + + #[tokio::test] + async fn pool_meta_v3_prepare_failure_drains_late_writes() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(4); + config[1][0].fault = WriteFault::Timeout; + config[3][0].gate = Some(gate.clone()); + let fixture = Fixture::new(config); + let mut state = fixture.state(); + let mut save = Box::pin( + fixture + .desired + .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]), + ); + tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") } + assert_eq!(fixture.trace.commit_count(), 0); + gate.add_permits(1); + assert!(save.await.is_err()); + assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4); + assert!(fixture.pools[3].inner.wrote.load(Ordering::SeqCst)); + assert!(state.ensure_write_safe("ambiguous prepare").is_err()); + } + + #[tokio::test] + async fn pool_meta_v3_rejections_restore_only_an_unmutated_phase() { + for previous in [None, Some("identity_cas")] { + let mut config = steps(4); + for step in &mut config { + step[0].fault = WriteFault::Reject; + } + let fixture = Fixture::new(config); + let state = fixture.state(); + let mut arm = state.arm_transaction(); + arm.phase = previous; + let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await; + assert!( + results + .into_iter() + .all(|result| matches!(result, Err(Error::PreconditionFailed))) + ); + assert_eq!(arm.phase, previous); + drop(arm); + assert_eq!(state.ensure_write_safe("all rejected").is_ok(), previous.is_none()); + } + let mut config = steps(4); + for step in &mut config[1..] { + step[0].fault = WriteFault::Reject; + } + let fixture = Fixture::new(config); + let state = fixture.state(); + let mut arm = state.arm_transaction(); + let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await; + assert!(results[0].is_ok()); + assert_eq!(arm.phase, Some("prepare_cas")); + drop(arm); + assert!(state.ensure_write_safe("one sibling wrote").is_err()); + } + + #[tokio::test] + async fn pool_meta_v3_parallel_errors_keep_original_source_pointers() { + let mut config = steps(4); + config[0][0].fault = WriteFault::Timeout; + config[1][0].fault = WriteFault::Timeout; + config[3][0].fault = WriteFault::Reject; + let fixture = Fixture::new(config); + let state = fixture.state(); + let mut arm = state.arm_transaction(); + let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await; + let err = results[0].as_ref().expect_err("first pool error stays first"); + let source = err + .pool_metadata_failure() + .expect("typed error") + .source + .as_ref() + .expect("original source"); + assert!(matches!(source.as_ref(), Error::Timeout)); + assert!(Arc::ptr_eq(source, arm.source.as_ref().expect("arm source"))); + drop(arm); + let blocked = state + .ensure_write_safe("after parallel errors") + .expect_err("recovery required"); + assert!(Arc::ptr_eq( + source, + blocked + .pool_metadata_failure() + .expect("typed latch") + .source + .as_ref() + .expect("latch source") + )); + } + + #[tokio::test] + async fn pool_meta_v3_retry_rejections_keep_the_previous_io_source() { + let mut config = steps(4); + config[0][0].fault = WriteFault::Reject; + config[1][0].fault = WriteFault::Timeout; + let first = Fixture::new(config); + let state = first.state(); + let mut arm = state.arm_transaction(); + let fence = PoolMetaPersistenceFence::Distributed(signal()); + let results = prepare(&first, &fence, &mut arm).await; + assert!(matches!(&results[0], Err(Error::PreconditionFailed))); + let original = arm.source.as_ref().expect("first attempt I/O source").clone(); + assert!(matches!(original.as_ref(), Error::Timeout)); + let mut config = steps(4); + for step in &mut config { + step[0].fault = WriteFault::Reject; + } + let retry = Fixture::new(config); + assert!( + prepare(&retry, &fence, &mut arm) + .await + .into_iter() + .all(|result| matches!(result, Err(Error::PreconditionFailed))) + ); + assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives phase retry"))); + implementation::save_pool_meta_object_cas( + retry.pools[0].clone(), + POOL_META_NAME, + retry.pending.clone(), + &retry.tokens()[0], + &fence, + "prepare_cas", + &mut arm, + ) + .await + .expect_err("serial retry rejected"); + assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives serial retry"))); + drop(arm); + let blocked = state + .ensure_write_safe("cancelled after retry rejection") + .expect_err("recovery required"); + assert!(Arc::ptr_eq( + &original, + blocked + .pool_metadata_failure() + .expect("typed latch") + .source + .as_ref() + .expect("source") + )); + } + + #[tokio::test] + async fn pool_meta_v3_cancelled_drain_does_not_replace_io_error_with_cas_rejection() { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(4); + config[0][0].gate = Some(gate.clone()); + config[2][0].gate = Some(gate); + config[1][0].fault = WriteFault::Timeout; + config[3][0].fault = WriteFault::Reject; + let fixture = Fixture::new(config); + let state = fixture.state(); + let mut arm = state.arm_transaction(); + let fence = PoolMetaPersistenceFence::Distributed(signal()); + let mut phase = Box::pin(prepare(&fixture, &fence, &mut arm)); + tokio::select! { _ = fixture.trace.wait_for(4, 2) => {}, _ = &mut phase => panic!("blocked phase finished") } + assert!(futures::poll!(phase.as_mut()).is_pending(), "drain remains parked on the two slow pools"); + drop(phase); + let source = arm.source.as_ref().expect("observed I/O source").clone(); + assert!(matches!(source.as_ref(), Error::Timeout)); + drop(arm); + let blocked = state + .ensure_write_safe("cancelled after mixed results") + .expect_err("recovery required"); + assert!(Arc::ptr_eq( + &source, + blocked + .pool_metadata_failure() + .expect("typed latch") + .source + .as_ref() + .expect("source") + )); + assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn pool_meta_v3_cancelled_prepare_commit_and_publication_keep_gate_blocked() { + for phase in [0, 1] { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(4); + for step in &mut config { + step[phase].gate = Some(gate.clone()); + } + let fixture = Fixture::new(config); + let mut state = fixture.state(); + let mut save = Box::pin( + fixture + .desired + .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]), + ); + tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") } + drop(save); + assert!(state.ensure_write_safe("cancelled phase").is_err()); + assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0); + assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0); + } + let fixture = Fixture::new(steps(4)); + let mut state = fixture.state(); + let outcome = fixture + .desired + .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]) + .await + .expect("durable save before publication"); + drop(outcome); + assert!(state.ensure_write_safe("publication was cancelled").is_err()); + assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn pool_meta_v3_fence_loss_blocks_queued_dispatch_and_marks_late_writes() { + let lock = NamespaceLock::with_clients_and_quorum( + "parallel-fence".to_owned(), + vec![Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))], + 1, + ); + let request = LockRequest::new( + ObjectKey::new(super::super::RUSTFS_META_BUCKET, POOL_META_NAME), + LockType::Exclusive, + "phase-test", + ) + .with_acquire_timeout(StdDuration::from_secs(1)) + .with_ttl(StdDuration::from_secs(1)) + .with_refresh_interval(StdDuration::from_secs(1)); + let guard = lock + .acquire_guard(&request) + .await + .expect("lease request") + .expect("lease acquired"); + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(6); + for step in &mut config { + step[0].gate = Some(gate.clone()); + } + let fixture = Fixture::new(config); + let state = fixture.state(); + let mut arm = state.arm_transaction(); + let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal()); + let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm)); + tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") } + tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified()) + .await + .expect("lease expires"); + gate.add_permits(6); + assert!(pending.await.into_iter().all(|result| result.is_err())); + assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4, "queued pools must not reach storage"); + assert!(fixture.pools[..4].iter().all(|pool| pool.inner.wrote.load(Ordering::SeqCst))); + assert_eq!(arm.phase, Some("prepare_cas")); + drop(arm); + assert!(state.ensure_write_safe("writes landed after lease loss").is_err()); + } + + #[tokio::test] + async fn pool_meta_v3_all_commit_errors_still_require_canonical_reread() { + for fault in [WriteFault::TimeoutAfterWrite, WriteFault::OmitWrite] { + let mut config = steps(4); + for step in &mut config { + step[1].fault = fault; + } + let fixture = Fixture::new(config); + let mut state = fixture.state(); + let result = fixture + .desired + .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]) + .await; + assert_eq!(fixture.trace.commit_count(), 4); + if matches!(fault, WriteFault::TimeoutAfterWrite) { + let outcome = result.expect("reread proves commits whose replies timed out"); + assert_eq!(outcome.committed.pools[0].last_update, fixture.desired.pools[0].last_update); + outcome.disarm(); + state + .ensure_write_safe("confirmed ambiguous commits") + .expect("confirmed transaction is safe"); + } else { + assert!(result.is_err(), "successful replies cannot replace canonical proof"); + assert!(state.ensure_write_safe("no committed canonical generation").is_err()); + } + } + } + + #[tokio::test] + async fn pool_meta_v3_none_and_legacy_paths_remain_serial() { + for (legacy, needs_repair) in [(false, false), (true, false), (false, true)] { + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut config = steps(4); + for step in &mut config { + step[0].gate = Some(gate.clone()); + step[1].gate = Some(gate.clone()); + } + let mut fixture = Fixture::new(config); + if legacy { + let mut meta = fixture.previous.meta.clone(); + meta.version = POOL_META_VERSION; + let data = meta.encode_config_data_for_v2_gate(true).expect("legacy metadata"); + for pool in &fixture.pools { + pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data.clone(); + } + fixture.desired.version = POOL_META_VERSION; + } + if needs_repair { + fixture.pools[3] + .inner + .stored + .lock() + .expect("fixture") + .as_mut() + .expect("replica") + .0 = fixture.pending.clone(); + } + let mut state = fixture.state(); + let mut save = Box::pin(fixture.desired.save_no_lock_armed( + fixture.pools.clone(), + &mut state, + if legacy || needs_repair { signal() } else { None }, + &[0], + )); + tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") } + assert!(futures::poll!(save.as_mut()).is_pending()); + assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 1); + gate.add_permits(16); + save.await.expect("serial save").disarm(); + assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 1); + } + } + + #[cfg(feature = "e2e-test-hooks")] + #[tokio::test] + async fn pool_meta_phase_barrier_leaves_noop_arm_for_the_next_changed_transaction() { + for case in [ + implementation::PoolMetaPhaseBarrierCase::PrepareSubset, + implementation::PoolMetaPhaseBarrierCase::PreparedAll, + implementation::PoolMetaPhaseBarrierCase::CommitOne, + implementation::PoolMetaPhaseBarrierCase::BeforePublish, + ] { + let fixture = Fixture::new(steps(4)); + let directory = tempfile::TempDir::new().expect("barrier directory"); + tokio::fs::write( + directory.path().join("arm.json"), + serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": case})).expect("arm"), + ) + .await + .expect("write arm"); + let noop = implementation::encode_pool_meta_v3_envelope(&fixture.previous.meta, fixture.revision, true, None) + .expect("new revision with unchanged pools"); + assert_ne!(noop, fixture.previous.canonical); + let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory( + directory.path().to_path_buf(), + &fixture.previous, + &fixture.previous.meta, + fixture.revision, + &noop, + ) + .await + .expect("unchanged pools must not consume the arm"); + assert!(barrier.is_none()); + assert!(!directory.path().join("claimed").exists()); + assert!(!directory.path().join("events.jsonl").exists()); + + let previous = implementation::PoolMetaCommittedCandidate { + meta: fixture.previous.meta.clone(), + revision: fixture.revision, + canonical: noop, + }; + let mut candidate = previous.meta.clone(); + let progress = candidate.pools[0].decommission.as_mut().expect("progress"); + progress.items_decommissioned += 1; + progress.bytes_done += 512; + let revision = implementation::PoolMetaRevision { + generation: previous.revision.generation + 1, + transaction_id: Some(uuid::Uuid::new_v4()), + ..previous.revision + }; + let durable = implementation::encode_pool_meta_v3_envelope(&candidate, revision, true, None) + .expect("changed progress without a timestamp change"); + let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory( + directory.path().to_path_buf(), + &previous, + &candidate, + revision, + &durable, + ) + .await + .expect("next changed transaction must bind") + .expect("bound barrier"); + assert_eq!(barrier.transaction_id, revision.transaction_id.expect("transaction")); + assert!(directory.path().join("claimed").is_file()); + let events = tokio::fs::read_to_string(directory.path().join("events.jsonl")) + .await + .expect("armed event"); + assert_eq!(events.lines().count(), 1); + let event: serde_json::Value = serde_json::from_str(events.trim()).expect("armed event JSON"); + assert_eq!(event["kind"], "armed"); + assert_eq!(event["previous"]["generation"], previous.revision.generation); + assert_eq!(event["candidate"]["generation"], revision.generation); + assert_ne!(event["previous"]["persisted_pools_sha256"], event["candidate"]["persisted_pools_sha256"]); + assert!( + implementation::PoolMetaPhaseBarrier::bind_in_directory( + directory.path().to_path_buf(), + &previous, + &candidate, + revision, + &durable, + ) + .await + .expect("already claimed arm") + .is_none() + ); + } + } + + #[cfg(feature = "e2e-test-hooks")] + #[tokio::test] + async fn pool_meta_phase_barrier_uses_persisted_responsibility_not_runtime_progress() { + let fixture = Fixture::new(steps(4)); + let directory = tempfile::TempDir::new().expect("barrier directory"); + tokio::fs::write( + directory.path().join("arm.json"), + serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": "prepare_subset"})).expect("arm"), + ) + .await + .expect("write arm"); + let mut candidate = fixture.previous.meta.clone(); + candidate.pools[0].decommission.as_mut().expect("progress").stage = "entry_started".to_owned(); + for changed in [false, true] { + if changed { + candidate.pools[0] + .decommission + .as_mut() + .expect("progress") + .queued_buckets + .push("new-responsibility".to_owned()); + } + let durable = + implementation::encode_pool_meta_v3_envelope(&candidate, fixture.revision, true, None).expect("candidate"); + let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory( + directory.path().to_path_buf(), + &fixture.previous, + &candidate, + fixture.revision, + &durable, + ) + .await + .expect("binding by persisted state"); + assert_eq!(barrier.is_some(), changed); + assert_eq!(directory.path().join("claimed").exists(), changed); + assert_eq!(directory.path().join("events.jsonl").exists(), changed); + } + } + + fn persisted_pools(meta: &PoolMeta) -> serde_json::Value { + serde_json::to_value( + meta.pools + .iter() + .map(implementation::PersistedPoolStatus::from) + .collect::>(), + ) + .expect("persisted pool state") + } + + #[tokio::test] + async fn pool_meta_v3_all_prepare_subsets_preserve_previous_snapshot() { + for previous_version in [POOL_META_VERSION, POOL_META_GENERATION_VERSION] { + for mask in 0usize..16 { + let mut fixture = Fixture::new(steps(4)); + if previous_version != POOL_META_GENERATION_VERSION { + fixture.previous.meta.version = previous_version; + fixture.previous.revision = implementation::PoolMetaRevision::legacy(previous_version); + fixture.revision.generation = 1; + fixture.previous.canonical = fixture + .previous + .meta + .encode_config_data_for_v2_gate(true) + .expect("legacy predecessor"); + fixture.pending = implementation::encode_pool_meta_v3_envelope( + &fixture.desired, + fixture.revision, + false, + Some(&fixture.previous), + ) + .expect("legacy pending envelope"); + } + let payloads = (0..4) + .map(|pool| { + if mask & (1 << pool) != 0 { + fixture.pending.clone() + } else { + fixture.previous.canonical.clone() + } + }) + .collect::>(); + let selected = implementation::select_pool_meta_replica( + payloads + .iter() + .cloned() + .map(implementation::decode_pool_meta_replica) + .collect(), + ) + .expect("every pending subset selects its predecessor"); + assert_eq!( + persisted_pools(&selected.meta), + persisted_pools(&fixture.previous.meta), + "mask={mask:04b}" + ); + assert_eq!(selected.revision, fixture.previous.revision); + if previous_version == POOL_META_GENERATION_VERSION { + for (pool, data) in fixture.pools.iter().zip(payloads) { + pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data; + } + let mut state = fixture.state(); + let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state) + .await + .expect("production recovery selection"); + let outcome = implementation::repair_pool_meta_transaction( + fixture.pools.clone(), + &mut state, + selected, + &PoolMetaPersistenceFence::Distributed(None), + ) + .await + .expect("production recovery repair"); + assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.previous.meta)); + outcome.disarm(); + let confirmed = + implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state) + .await + .expect("repaired reread"); + assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.previous.meta)); + assert!(!confirmed.replica_state.needs_repair); + } + } + } + } + + #[tokio::test] + async fn pool_meta_v3_every_sole_commit_position_recovers_new_snapshot() { + for committed_pool in 0..4 { + let fixture = Fixture::new(steps(4)); + for (index, pool) in fixture.pools.iter().enumerate() { + pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = if index == committed_pool { + fixture.durable.clone() + } else { + fixture.pending.clone() + }; + } + let mut state = fixture.state(); + let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state) + .await + .expect("sole commit must select new generation"); + assert_eq!(selected.revision, fixture.revision); + assert_eq!(persisted_pools(&selected.meta), persisted_pools(&fixture.desired)); + let outcome = implementation::repair_pool_meta_transaction( + fixture.pools.clone(), + &mut state, + selected, + &PoolMetaPersistenceFence::Distributed(None), + ) + .await + .expect("repair partial commit"); + assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.desired)); + outcome.disarm(); + let confirmed = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state) + .await + .expect("repaired sole commit reread"); + assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.desired)); + assert!(!confirmed.replica_state.needs_repair); + let fork = implementation::encode_pool_meta_v3_envelope( + &fixture.desired, + implementation::PoolMetaRevision { + transaction_id: Some(uuid::Uuid::new_v4()), + ..fixture.revision + }, + true, + None, + ) + .expect("same generation fork"); + assert!( + implementation::select_pool_meta_replica(vec![ + implementation::decode_pool_meta_replica(fixture.durable.clone()), + implementation::decode_pool_meta_replica(fork) + ]) + .is_err() + ); + } + } + } + #[tokio::test] async fn test_pool_meta_cas_deterministically_rejects_stale_writer() { let storage = Arc::new(PartialPoolMetaWriteStorage { diff --git a/crates/ecstore/src/core/pools_test.rs b/crates/ecstore/src/core/pools_test.rs index 43c557900..9e7e9390f 100644 --- a/crates/ecstore/src/core/pools_test.rs +++ b/crates/ecstore/src/core/pools_test.rs @@ -5256,16 +5256,35 @@ mod decommission_lock_order_tests { #[test] #[serial_test::serial] - fn scanner_backlog_native_replica_reconciles_capacity_and_cleans_source() { - run_large_stack_current_thread_async_test("scanner-backlog-reconcile", async || { + fn data_movement_existing_replica_reconciles_capacity_and_cleans_source() { + data_movement_existing_replica_reconciles_capacity_case(false); + } + + #[test] + #[serial_test::serial] + fn data_movement_existing_replica_outside_reservation_uses_reserved_target() { + data_movement_existing_replica_reconciles_capacity_case(true); + } + + fn data_movement_existing_replica_reconciles_capacity_case(existing_outside_reservation: bool) { + run_large_stack_current_thread_async_test("reserved-replica-reconcile", async move || { let (_temp_dirs, store, other_store) = test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await; - let object = "buckets/.scanner-pause-backlog.json"; + let object = "buckets/reserved-replica-routing.json"; let body = br#"{"schemaVersion":1,"generation":2}"#.to_vec(); let old_body = br#"{"schemaVersion":1,"generation":1}"#.to_vec(); let source_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(20); - let target_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10); - for (pool_index, payload, mod_time) in [(0, body.clone(), source_time), (2, old_body, target_time)] { + let target_time = source_time; + let target_pool_index = if existing_outside_reservation { 1 } else { 2 }; + let mut replicas = vec![(0, body.clone(), source_time), (target_pool_index, old_body, target_time)]; + if existing_outside_reservation { + replicas.push(( + 2, + br#"{"schemaVersion":1,"generation":3}"#.to_vec(), + source_time + time::Duration::seconds(10), + )); + } + for (pool_index, payload, mod_time) in replicas.iter().cloned() { store.pools[pool_index] .put_object( RUSTFS_META_BUCKET, @@ -5278,13 +5297,19 @@ mod decommission_lock_order_tests { }, ) .await - .expect("seed native scanner replicas with independent write times"); + .expect("seed existing replicas with independent write times"); } let layout = DecommissionErasureLayout { data: 1, parity: 0 }; let target_total = body.len() * 8; let capacities = vec![ DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len() * 2, body.len() * 2), - DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total), + DecommissionPoolCapacityInfo::for_test( + 1, + layout, + if existing_outside_reservation { target_total } else { 0 }, + target_total, + if existing_outside_reservation { 0 } else { target_total }, + ), DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0), ]; set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities.clone()]); @@ -5293,6 +5318,17 @@ mod decommission_lock_order_tests { .await .expect("activate the source reservation"); let owner = decommission_capacity_owner(&*store.pool_meta.read().await); + let reserved_snapshot = store.pool_meta.read().await.clone(); + let reservation = reserved_snapshot.pools[0] + .decommission + .as_ref() + .and_then(|info| info.capacity_reservation.as_ref()) + .expect("active source reservation"); + assert_eq!( + reservation.targets.iter().map(|target| target.pool_index).collect::>(), + vec![target_pool_index], + "the fixture must reserve exactly one target" + ); let source_reader = store.pools[0] .get_object_reader( RUSTFS_META_BUCKET, @@ -5314,12 +5350,91 @@ mod decommission_lock_order_tests { RUSTFS_META_BUCKET.to_string(), source_reader, None, - "scanner_backlog_conflict", + "reserved_replica_conflict", Some(owner), ) .await - .expect_err("a different older native ledger must retain its source and capacity intent"); + .expect_err("a different older existing record must retain its source and capacity intent"); assert!(conflict.to_string().contains("Precondition failed"), "unexpected conflict: {conflict}"); + let reserved_snapshot = store.pool_meta.read().await.clone(); + let mut selection_opts = ObjectOptions { + data_movement: true, + src_pool_idx: 0, + ..Default::default() + }; + assert_eq!( + store + .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true) + .await + .expect("selection without a capacity owner retains existing-replica routing"), + 2 + ); + owner.apply_to(&mut selection_opts); + for stale_owner in [ + DecommissionCapacityOwner { + owner_nonce: uuid::Uuid::new_v4(), + ..owner + }, + DecommissionCapacityOwner { + generation: owner.generation + 1, + ..owner + }, + ] { + let mut stale_opts = selection_opts.clone(); + stale_owner.apply_to(&mut stale_opts); + assert!( + matches!( + store + .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &stale_opts, true) + .await, + Err(crate::error::Error::DecommissionCapacityBlocked { .. }) + ), + "a stale owner must not fall back to another target" + ); + } + { + let mut meta = store.pool_meta.write().await; + meta.pools[0] + .decommission + .as_mut() + .unwrap() + .capacity_reservation + .as_mut() + .unwrap() + .expires_at = time::OffsetDateTime::now_utc() - time::Duration::seconds(1); + } + assert!( + matches!( + store + .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true) + .await, + Err(crate::error::Error::DecommissionCapacityBlocked { .. }) + ), + "an expired owner must not fall back to another target" + ); + *store.pool_meta.write().await = reserved_snapshot.clone(); + if !existing_outside_reservation { + { + let mut meta = store.pool_meta.write().await; + let target = &mut meta.pools[0] + .decommission + .as_mut() + .unwrap() + .capacity_reservation + .as_mut() + .unwrap() + .targets[0]; + target.consumed_physical_bytes = target.reserved_physical_bytes; + } + assert_eq!( + store + .select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true) + .await + .expect("an existing reserved replica can still be selected after capacity was consumed"), + target_pool_index + ); + *store.pool_meta.write().await = reserved_snapshot; + } let mut persisted = crate::core::pools::PoolMeta::default(); persisted .load_no_lock_from_replicas(store.pools.clone()) @@ -5336,11 +5451,24 @@ mod decommission_lock_order_tests { .pending_target_physical_bytes, body.len() ); - let previous = store.pools[2] + for (pool_index, payload, mod_time) in &replicas { + let mut reader = store.pools[*pool_index] + .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("a refused existing record replacement must preserve every replica"); + assert_eq!(reader.object_info.mod_time, Some(*mod_time)); + let mut actual = Vec::new(); + reader + .read_to_end(&mut actual) + .await + .expect("read the unchanged existing record"); + assert_eq!(&actual, payload); + } + let previous = store.pools[target_pool_index] .get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default()) .await - .expect("read the native writer's CAS revision"); - let replacement = store.pools[2] + .expect("read the existing writer's CAS revision"); + let replacement = store.pools[target_pool_index] .put_object( RUSTFS_META_BUCKET, object, @@ -5356,7 +5484,7 @@ mod decommission_lock_order_tests { }, ) .await - .expect("native scanner CAS converges the payload without a migration marker"); + .expect("existing CAS converges the payload without a migration marker"); assert!(!data_movement::is_owned_data_movement_target(&replacement)); *other_store.pool_meta.write().await = persisted; set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacities]); @@ -5374,7 +5502,7 @@ mod decommission_lock_order_tests { ) .await .expect("replica conflict recovery must be bounded") - .expect("identical native replica should finish migration on the reloaded node"); + .expect("identical existing replica should finish migration on the reloaded node"); let mut reconciled = crate::core::pools::PoolMeta::default(); reconciled .load_no_lock_from_replicas(other_store.pools.clone()) @@ -5404,14 +5532,14 @@ mod decommission_lock_order_tests { .await .expect_err("the source should be cleaned only after equivalent-target capacity reconciliation"); assert!(crate::error::is_err_object_not_found(&missing)); - let mut target_reader = other_store.pools[2] + let mut target_reader = other_store.pools[target_pool_index] .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default()) .await .expect("the surviving replica should remain readable"); assert_eq!( target_reader.object_info.mod_time, Some(target_time), - "recovery must not overwrite the native target" + "recovery must not overwrite the existing target" ); let mut actual = Vec::new(); target_reader @@ -5419,6 +5547,20 @@ mod decommission_lock_order_tests { .await .expect("read surviving ledger bytes"); assert_eq!(actual, body); + if existing_outside_reservation { + let (_, outside_body, outside_time) = replicas.last().expect("unreserved existing replica"); + let mut outside = other_store.pools[2] + .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("migration must leave the unreserved existing replica intact"); + assert_eq!(outside.object_info.mod_time, Some(*outside_time)); + let mut actual = Vec::new(); + outside + .read_to_end(&mut actual) + .await + .expect("read the untouched unreserved replica"); + assert_eq!(&actual, outside_body); + } }); } diff --git a/crates/ecstore/src/core/sets.rs b/crates/ecstore/src/core/sets.rs index ab51167e8..bfe88aae1 100644 --- a/crates/ecstore/src/core/sets.rs +++ b/crates/ecstore/src/core/sets.rs @@ -55,7 +55,7 @@ use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use std::{ collections::{HashMap, HashSet}, - sync::Arc, + sync::{Arc, Weak}, }; use tokio::sync::RwLock; use tokio::sync::broadcast::{Receiver, Sender}; @@ -219,7 +219,10 @@ impl Sets { let mut disk_set = Vec::with_capacity(set_count); - let lock_registry = runtime_sources::lock_registry(); + let pool_lockers = runtime_sources::lock_registry() + .as_ref() + .map(|registry| registry.clients_for_endpoints(endpoints.endpoints.as_ref())) + .unwrap_or_default(); for i in 0..set_count { let mut set_drive = Vec::with_capacity(set_drive_count); @@ -270,10 +273,6 @@ impl Sets { } } - let lockers = lock_registry - .as_ref() - .map(|registry| registry.clients_for_endpoints(&set_endpoints)) - .unwrap_or_default(); let set_disks = SetDisks::new_with_instance_ctx( runtime_sources::local_node_name().await, Arc::new(RwLock::new(set_drive)), @@ -283,7 +282,7 @@ impl Sets { pool_idx, set_endpoints, fm.clone(), - lockers, + pool_lockers.clone(), instance_ctx.clone(), ) .await; @@ -308,10 +307,9 @@ impl Sets { ctx: instance_ctx, }); - let asets = sets.clone(); - let rx1 = rx.resubscribe(); - tokio::spawn(async move { asets.monitor_and_connect_endpoints(rx1).await }); + let weak_sets = Arc::downgrade(&sets); + tokio::spawn(async move { Self::monitor_and_connect_endpoints_task(weak_sets, rx1).await }); Ok(sets) } @@ -326,12 +324,26 @@ impl Sets { &self.ctx } - pub async fn monitor_and_connect_endpoints(&self, mut rx: Receiver<()>) { - tokio::time::sleep(Duration::from_secs(5)).await; + async fn monitor_and_connect_endpoints_task(sets: Weak, mut rx: Receiver<()>) { + let startup_delay = tokio::time::sleep(Duration::from_secs(5)); + tokio::pin!(startup_delay); + + tokio::select! { + _ = &mut startup_delay => {} + _ = rx.recv() => { + warn!("monitor_and_connect_endpoints ctx cancelled"); + return; + } + } info!("start monitor_and_connect_endpoints"); - self.connect_disks().await; + let Some(current) = sets.upgrade() else { + warn!("monitor_and_connect_endpoints exit"); + return; + }; + current.connect_disks().await; + drop(current); // TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s let mut interval = tokio::time::interval(Duration::from_secs(15)); @@ -339,7 +351,10 @@ impl Sets { tokio::select! { _= interval.tick()=>{ // debug!("tick..."); - self.connect_disks().await; + let Some(current) = sets.upgrade() else { + break; + }; + current.connect_disks().await; interval.reset(); }, diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index b89f806bd..eb532c8dc 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -15,6 +15,7 @@ // #730: data-movement migration keeps staged cleanup helpers until copy paths converge. pub(crate) mod backpressure; +pub(crate) mod scanner_backlog; use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id}; use crate::error::{ @@ -984,24 +985,6 @@ fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: .is_some_and(|(source_time, target_time)| target_time > source_time) } -fn is_equivalent_scanner_backlog_replica(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool { - // Scanner publishes this exact payload to surviving sets with CAS. Each - // set assigns its own write time; that timestamp is not a ledger generation. - // Accept only an identical, known unversioned identity, never a different - // record based on timestamp ordering or a similarly named user object. - source.bucket == crate::disk::RUSTFS_META_BUCKET - && target.bucket == source.bucket - && source.name == "buckets/.scanner-pause-backlog.json" - && target.name == source.name - && is_unversioned_data_movement_object(source) - && is_unversioned_data_movement_object(target) - && !source.delete_marker - && source.mod_time.is_some() - && target.mod_time.is_some() - && source.etag.as_ref().is_some_and(|etag| !etag.is_empty()) - && is_equivalent_data_movement_object_identity(source, target, false, compare_part_checksums) -} - fn is_data_movement_upload_takeover_target(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool { let identity = data_movement_upload_identity(source); source.mod_time.is_some() @@ -1217,7 +1200,7 @@ struct SourceCleanupDeleteBarrierState { dead_code, reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)" )] -pub(crate) struct SourceCleanupDeleteBarrier { +pub struct SourceCleanupDeleteBarrier { state: Arc, } @@ -1231,7 +1214,7 @@ static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock Self { + pub fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(SourceCleanupDeleteBarrierState { bucket: bucket.to_string(), object: object.to_string(), @@ -1254,7 +1237,7 @@ impl SourceCleanupDeleteBarrier { Self { state } } - pub(crate) async fn wait_until_paused(&self) { + pub async fn wait_until_paused(&self) { tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified()) .await .expect("source cleanup should reach the pre-delete barrier"); @@ -1270,7 +1253,7 @@ impl SourceCleanupDeleteBarrier { self.state.is_paused.load(Ordering::Acquire) } - pub(crate) fn release(&self) { + pub fn release(&self) { self.state.release.notify_one(); } } @@ -1449,7 +1432,8 @@ fn resolve_data_movement_overwrite_resume_result_for( target_pool_idx: usize, compare_part_checksums: bool, ) -> Result { - if !should_check_data_movement_overwrite_resume(err) + if scanner_backlog::is_scanner_pause_backlog(&source.bucket, &source.name) + || !should_check_data_movement_overwrite_resume(err) || !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) { return Ok(false); @@ -1471,9 +1455,7 @@ fn resolve_data_movement_overwrite_resume_result_for( return Ok(true); } - Ok(matches!(err, Error::PreconditionFailed) - && (is_equivalent_scanner_backlog_replica(source, &target, compare_part_checksums) - || is_superseding_unversioned_data_movement_object(source, &target))) + Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target)) } #[derive(Clone, Copy)] @@ -1521,9 +1503,27 @@ fn data_movement_part_stage_error( bucket: &str, object: &str, part_number: usize, - err: impl std::fmt::Display, + err: Error, ) -> Error { - Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}")) + let rendered = format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}"); + if matches!(&err, Error::DecommissionCapacityBlocked { .. }) { + return data_movement_context_error(rendered, err); + } + // A missing target part is not evidence that the source can be deleted. + // Keep other part errors opaque to the source-cleanup classifiers. + Error::other(rendered) +} + +#[cfg(test)] +pub(crate) fn data_movement_part_stage_error_for_test( + op_label: &str, + stage: &str, + bucket: &str, + object: &str, + part_number: usize, + err: Error, +) -> Error { + data_movement_part_stage_error(op_label, stage, bucket, object, part_number, err) } fn is_data_movement_part_read_error(err: &Error) -> bool { @@ -1628,6 +1628,9 @@ async fn migrate_object_inner( capacity_owner: Option, mutation_fence: Option, ) -> Result<()> { + if scanner_backlog::is_scanner_pause_backlog(&bucket, &rd.object_info.name) { + return Err(Error::other("scanner pause backlog requires native retirement handoff")); + } let mut mutation_fence = mutation_fence; let object_info = rd.object_info.clone(); let capacity_owner = capacity_owner.map(|owner| { @@ -2428,8 +2431,15 @@ mod tests { let err = data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown); let message = err.to_string(); - assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7")); - assert!(message.contains(Error::SlowDown.to_string().as_str())); + assert_eq!( + message, + Error::other(format!( + "rebalance_object: put_object_part failed for bucket-a/object-a part 7: {}", + Error::SlowDown + )) + .to_string() + ); + assert!(data_movement_stage_source(&err).is_none()); } #[test] @@ -3329,16 +3339,25 @@ mod tests { } #[test] - fn test_scanner_backlog_resume_accepts_identical_native_replica_with_older_write_time() { + fn test_scanner_backlog_resume_requires_native_cohort_proof_even_for_identical_payload() { let (source, target) = scanner_backlog_replica_pair(); assert!(!is_owned_data_movement_target(&target), "native scanner writes are not migration copies"); assert!(!is_equivalent_data_movement_object(&source, &target)); assert!( - scanner_backlog_precondition_resumes(&source, target), - "identical ledger payloads have replica-local write times, not distinct committed generations" + !scanner_backlog_precondition_resumes(&source, target), + "a single identical replica cannot prove native cohort authority" ); } + #[test] + fn test_scanner_backlog_resume_rejects_newer_timestamp_and_full_single_replica_identity() { + let (source, mut target) = scanner_backlog_replica_pair(); + target.mod_time = source.mod_time.map(|time| time + time::Duration::SECOND); + target.etag = Some("different-native-ledger".to_string()); + assert!(!scanner_backlog_precondition_resumes(&source, target)); + assert!(!scanner_backlog_precondition_resumes(&source, source.clone())); + } + #[test] fn test_scanner_backlog_resume_rejects_changed_payload_or_metadata() { let (source, target) = scanner_backlog_replica_pair(); diff --git a/crates/ecstore/src/data_movement/scanner_backlog.rs b/crates/ecstore/src/data_movement/scanner_backlog.rs new file mode 100644 index 000000000..5a5d819cc --- /dev/null +++ b/crates/ecstore/src/data_movement/scanner_backlog.rs @@ -0,0 +1,292 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::disk::RUSTFS_META_BUCKET; +use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found}; +use crate::object_api::ObjectOptions; +use crate::object_api::{ObjectInfo, PutObjReader, WriteCompletion}; +use crate::set_disk::SetDisks; +use crate::storage_api_contracts::object::HTTPPreconditions; +use crate::storage_api_contracts::object::ObjectIO as _; +use futures::future::join_all; +use http::HeaderMap; +use std::sync::{Arc, OnceLock}; +use tokio::io::AsyncReadExt; + +pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024; +pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json"; + +/// A bounded, storage-fenced native replica. Only a confirmed missing object +/// has no payload; read failures never enter the Scanner verifier. +pub struct ScannerPauseBacklogRetirementReplica { + pub pool_index: usize, + pub set_index: usize, + pub data: Option>, +} + +/// Native records for a membership handoff. Existing durable ledgers are +/// preserved; an empty native bootstrap may initialize its first ledger. +pub struct ScannerPauseBacklogRetirementPlan { + pub seed_record: Option>, + pub commit_record: Vec, + pub stable_record: Vec, +} + +pub type ScannerPauseBacklogRetirementPlanner = + fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result, String>; + +static RETIREMENT_PLANNER: OnceLock = OnceLock::new(); + +/// Install the stateless native record planner before storage starts workers. +/// The scanner runtime switch does not control this storage safety check. +pub fn register_scanner_pause_backlog_retirement_planner(planner: ScannerPauseBacklogRetirementPlanner) { + RETIREMENT_PLANNER.get_or_init(|| planner); +} + +pub(crate) fn is_scanner_pause_backlog(bucket: &str, object: &str) -> bool { + bucket == RUSTFS_META_BUCKET && object == SCANNER_PAUSE_BACKLOG_PATH +} + +pub(crate) struct ScannerPauseBacklogRetirementRead { + pub replica: ScannerPauseBacklogRetirementReplica, + pub etag: Option, +} + +impl ScannerPauseBacklogRetirementRead { + pub(crate) fn preconditions(&self) -> HTTPPreconditions { + match &self.etag { + Some(etag) => HTTPPreconditions { + if_match: Some(etag.clone()), + ..Default::default() + }, + None => HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + } + } +} + +async fn read_replica(set: Arc) -> Result { + let mut replica = ScannerPauseBacklogRetirementReplica { + pool_index: set.pool_index, + set_index: set.set_index, + data: None, + }; + let reader = match set + .get_object_reader( + RUSTFS_META_BUCKET, + SCANNER_PAUSE_BACKLOG_PATH, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader, + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => { + return Ok(ScannerPauseBacklogRetirementRead { replica, etag: None }); + } + Err(err) => return Err(err), + }; + let info = &reader.object_info; + if info.version_id.is_some_and(|version| !version.is_nil()) + || info.delete_marker + || info.is_dir + || info.etag.as_ref().is_none_or(String::is_empty) + || info.size < 0 + || info.size > MAX_SCANNER_PAUSE_BACKLOG_BYTES as i64 + { + return Err(Error::other("scanner pause backlog retirement found an unsupported replica identity")); + } + let etag = info.etag.clone(); + let expected_size = info.size as usize; + let mut data = Vec::new(); + reader + .take(MAX_SCANNER_PAUSE_BACKLOG_BYTES + 1) + .read_to_end(&mut data) + .await?; + if data.len() != expected_size || data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize { + return Err(Error::other("scanner pause backlog retirement replica has an invalid payload length")); + } + replica.data = Some(data); + Ok(ScannerPauseBacklogRetirementRead { replica, etag }) +} + +/// The caller retains the fixed object write lock and durable topology read +/// fence through both this snapshot and physical source cleanup. +pub(crate) async fn read_scanner_pause_backlog_retirement_replicas( + source_pool_index: usize, + source_set_index: usize, + sets: Vec>, +) -> Result> { + let replicas = join_all(sets.into_iter().map(read_replica)) + .await + .into_iter() + .collect::>>()?; + if !replicas.iter().any(|read| { + read.replica.pool_index == source_pool_index && read.replica.set_index == source_set_index && read.replica.data.is_some() + }) { + return Err(Error::other("scanner pause backlog retirement current source replica is missing")); + } + Ok(replicas) +} + +pub(crate) fn plan_scanner_pause_backlog_retirement( + source_pool_index: usize, + replicas: &[ScannerPauseBacklogRetirementRead], +) -> Result> { + let planner = RETIREMENT_PLANNER + .get() + .ok_or_else(|| Error::other("scanner pause backlog native retirement planner is unavailable"))?; + let snapshots = replicas + .iter() + .map(|read| ScannerPauseBacklogRetirementReplica { + pool_index: read.replica.pool_index, + set_index: read.replica.set_index, + data: read.replica.data.clone(), + }) + .collect::>(); + planner(source_pool_index, &snapshots).map_err(Error::other) +} + +/// The native writer and retirement handoff use the same conditional, full-tail +/// write. Their callers retain object and durable membership fences until return. +pub(crate) async fn persist_native_scanner_pause_backlog_replica( + set: Arc, + data: Vec, + preconditions: HTTPPreconditions, + mut opts: ObjectOptions, + _phase: &'static str, +) -> Result { + if data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize { + return Err(Error::other("scanner pause backlog exceeds its size bound")); + } + opts.max_parity = true; + opts.write_completion = WriteCompletion::TailDrained; + opts.http_preconditions = Some(preconditions); + #[cfg(feature = "test-util")] + let fault = test_util::matching_write(&set, _phase)?; + let result = set + .put_object(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, &mut PutObjReader::from_vec(data), &opts) + .await; + #[cfg(feature = "test-util")] + if result.is_ok() + && let Some(fault) = fault + { + fault.arrived.notify_one(); + fault.release.notified().await; + } + result +} + +#[cfg(feature = "test-util")] +pub mod test_util { + use super::*; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::Notify; + + #[derive(Debug, thiserror::Error)] + #[error("injected native scanner backlog {phase} write failure")] + struct InjectedWriteFailure { + phase: &'static str, + } + + pub(super) struct WriteFault { + set: Arc, + phase: &'static str, + remaining: AtomicUsize, + fail_before_write: bool, + pub(super) arrived: Notify, + pub(super) release: Notify, + } + + static WRITE_FAULTS: Mutex>> = Mutex::new(Vec::new()); + + /// Scope a one-shot fault to the actual set instance, so other stores and + /// concurrent tests keep using the ordinary native persistence path. + pub struct NativeScannerPauseBacklogWriteFault { + state: Arc, + } + + impl NativeScannerPauseBacklogWriteFault { + fn install(set: Arc, phase: &'static str, nth: usize, fail_before_write: bool) -> Self { + assert!(nth > 0); + let state = Arc::new(WriteFault { + set, + phase, + remaining: AtomicUsize::new(nth), + fail_before_write, + arrived: Notify::new(), + release: Notify::new(), + }); + let mut faults = WRITE_FAULTS.lock().unwrap(); + assert!( + !faults + .iter() + .any(|fault| Arc::ptr_eq(&fault.set, &state.set) && fault.phase == phase) + ); + faults.push(Arc::clone(&state)); + Self { state } + } + + pub fn fail_before_write(set: Arc, phase: &'static str, nth: usize) -> Self { + Self::install(set, phase, nth, true) + } + + pub fn pause_after_write(set: Arc, phase: &'static str) -> Self { + Self::install(set, phase, 1, false) + } + + pub async fn wait_until_paused(&self) { + self.state.arrived.notified().await; + } + + pub fn release(&self) { + self.state.release.notify_one(); + } + } + + impl Drop for NativeScannerPauseBacklogWriteFault { + fn drop(&mut self) { + self.release(); + WRITE_FAULTS.lock().unwrap().retain(|fault| !Arc::ptr_eq(fault, &self.state)); + } + } + + pub(super) fn matching_write(set: &Arc, phase: &'static str) -> Result>> { + let fault = WRITE_FAULTS + .lock() + .unwrap() + .iter() + .find(|fault| Arc::ptr_eq(&fault.set, set) && fault.phase == phase) + .cloned(); + let Some(fault) = fault else { return Ok(None) }; + if fault + .remaining + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1)) + != Ok(1) + { + return Ok(None); + } + if fault.fail_before_write { + return Err(Error::other(InjectedWriteFailure { phase })); + } + Ok(Some(fault)) + } +} diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index b829208ea..49ee5ced0 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -3397,6 +3397,7 @@ mod tests { scan_plan_digest: Some([1; 32]), complete: false, tombstone: false, + segment_invalidation_proof: None, }]; partial.buckets_usage.insert( "bucket".to_string(), @@ -3469,6 +3470,7 @@ mod tests { scan_plan_digest: Some([1; 32]), complete: true, tombstone: false, + segment_invalidation_proof: None, }], ..Default::default() }; diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index f78611ed9..108343c90 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -219,6 +219,7 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r } } +#[cfg(test)] async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> { write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await } @@ -250,6 +251,7 @@ async fn write_delete_rollback_file( Ok(()) } +#[cfg(test)] async fn restore_metadata_backup( object_dir: &Path, xl_path: &Path, @@ -277,15 +279,6 @@ async fn restore_metadata_backup_with_namespace_owner( Ok(()) } -async fn restore_delete_rollback( - object_dir: &Path, - xl_path: &Path, - rollback_dir: Uuid, - publication_root: &os::PublicationRoot, -) -> Result<()> { - restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, None).await -} - async fn restore_delete_rollback_with_namespace_owner( object_dir: &Path, xl_path: &Path, @@ -5857,6 +5850,8 @@ impl LocalDisk { check_path_length(file_path.to_string_lossy().as_ref())?; let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]); + let namespace_owner: Option> = + Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await); if opts.old_data_dir.is_some() && opts.undo_write { return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await; } @@ -6718,6 +6713,8 @@ impl LocalDisk { async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; + let object_path = self.get_object_path(volume, path)?; + let namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; let object_dir = xlpath .parent() @@ -6727,10 +6724,24 @@ impl LocalDisk { && opts.undo_write { if opts.undo_delete { - return restore_delete_rollback(object_dir, &xlpath, rollback_dir, &self.publication_root).await; + return restore_delete_rollback_with_namespace_owner( + object_dir, + &xlpath, + rollback_dir, + &self.publication_root, + namespace_owner.clone(), + ) + .await; } - return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await; + return restore_metadata_backup_with_namespace_owner( + object_dir, + &xlpath, + rollback_dir, + &self.publication_root, + namespace_owner.clone(), + ) + .await; } let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await { @@ -6742,7 +6753,14 @@ impl LocalDisk { return Err(DiskError::FileNotFound); }; return self - .write_missing_delete_marker(volume, path, delete_marker, object_dir, opts.old_data_dir, None) + .write_missing_delete_marker( + volume, + path, + delete_marker, + object_dir, + opts.old_data_dir, + namespace_owner.clone(), + ) .await; } Err(err) => return Err(err), @@ -6758,7 +6776,8 @@ impl LocalDisk { let rollback_dir = opts.old_data_dir; let mut reserved_version_delete = false; if let Some(rollback_dir) = rollback_dir { - write_metadata_rollback_backup(object_dir, rollback_dir, &data).await?; + write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone()) + .await?; } for fi in fis.iter() { @@ -6772,13 +6791,16 @@ impl LocalDisk { if reserved_version_delete && let Some(rollback_dir) = rollback_dir { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_metadata_update", - err, + DeleteRollbackFailure { + stage: "delete_versions_metadata_update", + error: err, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -6791,7 +6813,7 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_metadata_update", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) @@ -6808,13 +6830,16 @@ impl LocalDisk { Err(err) => { if reserved_version_delete && let Some(rollback_dir) = rollback_dir { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_data_path", - err, + DeleteRollbackFailure { + stage: "delete_versions_data_path", + error: err, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -6827,7 +6852,7 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_data_path", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) @@ -6840,13 +6865,16 @@ impl LocalDisk { let err: DiskError = to_file_error(err).into(); if reserved_version_delete { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_rollback_dir", - err, + DeleteRollbackFailure { + stage: "delete_versions_rollback_dir", + error: err, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -6859,23 +6887,29 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_rollback_dir", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) .await); } - let reserved = match self.reserve_version_delete(volume, path, dir, rollback_dir).await { + let reserved = match self + .reserve_version_delete_with_namespace_owner(volume, path, dir, rollback_dir, namespace_owner.clone()) + .await + { Ok(reserved) => reserved, Err(err) => { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_reserve_data", - err, + DeleteRollbackFailure { + stage: "delete_versions_reserve_data", + error: err, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -6883,11 +6917,12 @@ impl LocalDisk { reserved_version_delete |= reserved; let rollback_data_path = rollback_path.join(dir.to_string()); if !reserved - && let Err(err) = rename_all_ignore_missing_source( + && let Err(err) = os::rename_all_ignore_missing_source_with_owner( &dir_path, &rollback_data_path, &rollback_path, &self.publication_root, + namespace_owner.clone(), ) .await { @@ -6900,7 +6935,7 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_stage_data", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) @@ -6909,13 +6944,16 @@ impl LocalDisk { if should_fail_after_delete_data_staged(path) { if reserved_version_delete { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_test_after_stage", - DiskError::Unexpected, + DeleteRollbackFailure { + stage: "delete_versions_test_after_stage", + error: DiskError::Unexpected, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -6928,13 +6966,15 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_test_after_stage", error: DiskError::Unexpected, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) .await); } - } else if let Err(err) = self.move_to_trash(&dir_path, true, false).await + } else if let Err(err) = self + .move_to_trash_with_namespace_owner(&dir_path, true, false, namespace_owner.clone()) + .await && !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound) { return Err(err); @@ -6949,16 +6989,22 @@ impl LocalDisk { // Remove xl.meta when no versions remain if fm.versions.is_empty() { - if let Err(err) = self.delete_file(&volume_dir, &xlpath, true, false).await { + if let Err(err) = self + .delete_file_with_namespace_owner(&volume_dir, &xlpath, true, false, namespace_owner.clone()) + .await + { if reserved_version_delete && let Some(rollback_dir) = rollback_dir { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_commit_delete", - err, + DeleteRollbackFailure { + stage: "delete_versions_commit_delete", + error: err, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -6971,7 +7017,7 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_commit_delete", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) @@ -6979,10 +7025,22 @@ impl LocalDisk { } if reserved_version_delete && let Some(rollback_dir) = rollback_dir - && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await + && let Err(err) = self + .commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone()) + .await { return Err(self - .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err) + .abort_reserved_version_delete_with_failure( + object_dir, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_versions_commit_intent", + error: err, + namespace_owner: namespace_owner.clone(), + }, + ) .await); } if should_fail_after_delete_commit(self.root.as_path(), path) { @@ -6999,13 +7057,16 @@ impl LocalDisk { let err: DiskError = err.into(); if reserved_version_delete && let Some(rollback_dir) = rollback_dir { return Err(self - .abort_reserved_version_delete( + .abort_reserved_version_delete_with_failure( object_dir, rollback_dir, volume, path, - "delete_versions_metadata_encode", - err, + DeleteRollbackFailure { + stage: "delete_versions_metadata_encode", + error: err, + namespace_owner: namespace_owner.clone(), + }, ) .await); } @@ -7018,7 +7079,7 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_metadata_encode", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) @@ -7027,12 +7088,28 @@ impl LocalDisk { }; if let Err(err) = self - .write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &buf, true) + .write_all_meta_with_namespace_owner( + volume, + format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), + &buf, + true, + namespace_owner.clone(), + ) .await { if reserved_version_delete && let Some(rollback_dir) = rollback_dir { return Err(self - .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_write", err) + .abort_reserved_version_delete_with_failure( + object_dir, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_versions_commit_write", + error: err, + namespace_owner: namespace_owner.clone(), + }, + ) .await); } return Err(restore_delete_rollback_after_error( @@ -7044,7 +7121,7 @@ impl LocalDisk { DeleteRollbackFailure { stage: "delete_versions_commit_write", error: err, - namespace_owner: None, + namespace_owner: namespace_owner.clone(), }, &self.publication_root, ) @@ -7053,10 +7130,22 @@ impl LocalDisk { if reserved_version_delete && let Some(rollback_dir) = rollback_dir - && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await + && let Err(err) = self + .commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone()) + .await { return Err(self - .abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err) + .abort_reserved_version_delete_with_failure( + object_dir, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_versions_commit_intent", + error: err, + namespace_owner: namespace_owner.clone(), + }, + ) .await); } @@ -7067,6 +7156,107 @@ impl LocalDisk { Ok(()) } + async fn reconcile_transition_state_metadata( + &self, + volume: &str, + object: &str, + version_id: Option, + condition: &super::TransitionStateReconcileCondition, + namespace_owner: Option>, + authority: Arc, + ) -> Result<()> { + condition.target.validate()?; + if !authority.is_current() + || [ + condition.expected_metadata_digest.as_str(), + condition.unchanged_metadata_digest.as_str(), + ] + .iter() + .any(|digest| { + digest.len() != 64 + || !digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) + { + return Err(DiskError::OutdatedXLMeta); + } + let metadata_path = format!("{object}/{STORAGE_FORMAT_FILE}"); + let original = self.read_all(volume, &metadata_path).await?; + let original_digest = rustfs_utils::crypto::hex(::digest(&original)); + let mut metadata = FileMeta::load(&original)?; + let (_, selected) = metadata.find_version(version_id)?; + if selected.into_fileinfo(volume, object, true)?.transition_tier != condition.tier { + return Err(DiskError::OutdatedXLMeta); + } + let generation = metadata.transition_reconcile_generation(version_id)?; + if rustfs_utils::crypto::hex(::digest(&generation)) != condition.unchanged_metadata_digest { + return Err(DiskError::OutdatedXLMeta); + } + let changed = metadata.reconcile_transition_state(version_id, &condition.target)?; + if !changed { + return Ok(()); + } + if condition.verify_only || original_digest != condition.expected_metadata_digest { + return Err(DiskError::OutdatedXLMeta); + } + // fsync_dir_std is a no-op outside Unix, so those platforms cannot + // yet prove this repair's durable publication requirement. + if !cfg!(unix) || !effective_durability(volume).syncs_commit_metadata() { + return Err(DiskError::other( + "transition reconciliation requires Unix directory sync and enabled bucket metadata durability", + )); + } + // An outstanding rollback can still restore an older whole xl.meta. + // Its preparation and execution share this mutation domain; refuse + // repair until that transaction has settled and removed its backup. + let mut entries = fs::read_dir(self.io_get_object_path(volume, object)?) + .await + .map_err(to_file_error)?; + let mut remaining = 4096usize; + while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? { + remaining = remaining.checked_sub(1).ok_or(DiskError::OutdatedXLMeta)?; + if Uuid::parse_str(&entry.file_name().to_string_lossy()).is_err() { + continue; + } + for marker in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] { + if fs::try_exists(entry.path().join(marker)).await.map_err(to_file_error)? { + return Err(DiskError::OutdatedXLMeta); + } + } + } + let replacement = metadata.marshal_msg()?; + let tmp_volume = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?; + let tmp_file = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &Uuid::new_v4().to_string())?; + // Admission above requires metadata durability. Keep rename and its + // directory sync in the same owned executor even after cancellation. + self.write_all_internal(&tmp_file, InternalBuf::Ref(&replacement), SyncMode::FileOnly, &tmp_volume) + .await?; + if crash_inject::should_crash_at(CrashPoint::MetaWriteAfterTmpBeforeRename, &metadata_path) { + return Err(DiskError::Unexpected); + } + os::rename_reconciled_metadata( + tmp_file, + self.io_get_object_path(volume, &metadata_path)?, + self.io_get_bucket_path(volume)?, + self.publication_root.clone(), + namespace_owner.clone(), + authority, + ) + .await?; + // Keep the same mutation lease through strong readback. Response loss + // leaves a monotonic subset for the coordinator's exact-copy retry. + let committed = self.read_all(volume, &metadata_path).await?; + let mut committed = FileMeta::load(&committed)?; + if committed.reconcile_transition_state(version_id, &condition.target)? + || committed.transition_reconcile_generation(version_id)? != generation + { + return Err(DiskError::OutdatedXLMeta); + } + Ok(()) + } + + #[cfg(test)] async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> { self.write_all_meta_with_namespace_owner(volume, path, buf, sync, None).await } @@ -8325,6 +8515,7 @@ impl LocalDisk { Ok(Arc::new(QuotaMutationFenceClaim { state })) } + #[cfg(test)] async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result { self.reserve_version_delete_with_namespace_owner(volume, object, data_dir, rollback_dir, None) .await @@ -8377,6 +8568,7 @@ impl LocalDisk { Ok(true) } + #[cfg(test)] async fn commit_reserved_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<()> { self.commit_reserved_version_delete_with_namespace_owner(volume, object, rollback_dir, None) .await @@ -8468,29 +8660,6 @@ impl LocalDisk { first_err.map_or(Ok(found), Err) } - async fn abort_reserved_version_delete( - &self, - object_dir: &Path, - rollback_dir: Uuid, - volume: &str, - object: &str, - stage: &'static str, - err: DiskError, - ) -> DiskError { - self.abort_reserved_version_delete_with_failure( - object_dir, - rollback_dir, - volume, - object, - DeleteRollbackFailure { - stage, - error: err, - namespace_owner: None, - }, - ) - .await - } - async fn abort_reserved_version_delete_with_failure( &self, object_dir: &Path, @@ -10035,6 +10204,23 @@ impl DiskAPI for LocalDisk { #[tracing::instrument(level = "trace", skip_all)] async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> { + let object_path = self.get_object_path(volume, path)?; + if let Some(condition) = &opts.transition_reconcile { + if !fi.metadata.is_empty() || opts.no_persistence || opts.replace_user_metadata { + return Err(DiskError::FileCorrupt); + } + let authority = + crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority::for_disk( + condition, + ) + .await?; + let owner: Option> = Some(authority.clone()); + let owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, owner).await); + return self + .reconcile_transition_state_metadata(volume, path, fi.version_id, condition, owner, authority) + .await; + } + let namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); if !fi.metadata.is_empty() { let file_path = self.io_get_object_path(volume, path)?; @@ -10062,7 +10248,13 @@ impl DiskAPI for LocalDisk { let wbuf = xl_meta.marshal_msg()?; return self - .write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &wbuf, !opts.no_persistence) + .write_all_meta_with_namespace_owner( + volume, + format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), + &wbuf, + !opts.no_persistence, + namespace_owner, + ) .await; } @@ -10070,7 +10262,10 @@ impl DiskAPI for LocalDisk { } async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { - self.write_metadata_with_namespace_owner(volume, path, fi, None).await + let object_path = self.get_object_path(volume, path)?; + let namespace_owner: Option> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await); + self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner) + .await } #[tracing::instrument(level = "trace", skip_all)] @@ -22637,6 +22832,192 @@ mod test { ); } + #[cfg(unix)] + #[tokio::test] + async fn conditional_mrf_manifest_dir_fsync_failure_keeps_recovery_anchors() { + use tempfile::tempdir; + + const MRF_COMMIT_MANIFEST_SLOT_0: &str = ".heal-mrf-commit.0.bin"; + const MRF_COMMIT_MANIFEST_SLOT_1: &str = ".heal-mrf-commit.1.bin"; + const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin"; + + let _mode = durability_mode_override::set(DurabilityMode::Relaxed); + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let previous_manifest = Bytes::from_static(b"mrf-committed-manifest-v1"); + let successor_manifest = Bytes::from_static(b"mrf-committed-manifest-v2"); + let legacy_journal = Bytes::from_static(b"legacy-mrf-journal-records"); + + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0, None, Some(previous_manifest.clone()),) + .await + .expect("previous MRF manifest should commit"), + ConditionalFileUpdate::Updated + ); + disk.write_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy_journal.clone()) + .await + .expect("legacy MRF journal should be retained"); + + let manifest_path = disk + .get_object_path(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0) + .expect("MRF manifest path should resolve"); + let parent = manifest_path.parent().expect("MRF manifest path should have a parent"); + assert!( + os::fsync_dir_recorder::was_fsynced(parent), + "system metadata MRF manifest publication must fsync the metadata directory even under relaxed durability" + ); + os::fsync_dir_recorder::set_failure(parent, ErrorKind::Other); + + let err = disk + .compare_and_update_file( + RUSTFS_META_BUCKET, + MRF_COMMIT_MANIFEST_SLOT_0, + Some(previous_manifest.clone()), + Some(successor_manifest), + ) + .await + .expect_err("directory fsync failure must fail the MRF manifest successor commit"); + assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::Other)); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0) + .await + .expect("previous committed MRF manifest should remain readable after rollback"), + previous_manifest + ); + + os::fsync_dir_recorder::set_failure(parent, ErrorKind::Other); + let err = disk + .compare_and_update_file( + RUSTFS_META_BUCKET, + MRF_COMMIT_MANIFEST_SLOT_1, + None, + Some(Bytes::from_static(b"first-successor-manifest")), + ) + .await + .expect_err("directory fsync failure must fail first MRF manifest commit"); + assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::Other)); + assert!( + matches!( + disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_1).await, + Err(DiskError::FileNotFound) + ), + "uncommitted first MRF manifest must be removed when no committed anchor exists" + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("legacy MRF journal should remain readable after failed manifest publication"), + legacy_journal + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn conditional_mrf_manifest_storage_full_keeps_recovery_anchors() { + use tempfile::tempdir; + + const MRF_COMMIT_MANIFEST_SLOT_0: &str = ".heal-mrf-commit.0.bin"; + const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin"; + + let _mode = durability_mode_override::set(DurabilityMode::Relaxed); + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let previous_manifest = Bytes::from_static(b"mrf-committed-manifest-v1"); + let successor_manifest = Bytes::from_static(b"mrf-committed-manifest-v2"); + let legacy_journal = Bytes::from_static(b"legacy-mrf-journal-records"); + + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0, None, Some(previous_manifest.clone()),) + .await + .expect("previous MRF manifest should commit"), + ConditionalFileUpdate::Updated + ); + disk.write_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy_journal.clone()) + .await + .expect("legacy MRF journal should be retained"); + + let manifest_path = disk + .get_object_path(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0) + .expect("MRF manifest path should resolve"); + let parent = manifest_path.parent().expect("MRF manifest path should have a parent"); + os::fsync_dir_recorder::set_failure(parent, ErrorKind::StorageFull); + + let err = disk + .compare_and_update_file( + RUSTFS_META_BUCKET, + MRF_COMMIT_MANIFEST_SLOT_0, + Some(previous_manifest.clone()), + Some(successor_manifest), + ) + .await + .expect_err("storage-full fsync failure must fail the MRF manifest successor commit"); + assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::StorageFull)); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0) + .await + .expect("previous committed MRF manifest should remain readable after storage-full rollback"), + previous_manifest + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("legacy MRF journal should remain readable after storage-full manifest publication failure"), + legacy_journal + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn conditional_mrf_manifest_storage_full_delete_keeps_recovery_anchors() { + use tempfile::tempdir; + + const MRF_COMMIT_MANIFEST_SLOT_0: &str = ".heal-mrf-commit.0.bin"; + const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin"; + + let _mode = durability_mode_override::set(DurabilityMode::Relaxed); + let dir = tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + let committed_manifest = Bytes::from_static(b"mrf-committed-manifest-v1"); + let legacy_journal = Bytes::from_static(b"legacy-mrf-journal-records"); + + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0, None, Some(committed_manifest.clone()),) + .await + .expect("committed MRF manifest should publish"), + ConditionalFileUpdate::Updated + ); + disk.write_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy_journal.clone()) + .await + .expect("legacy MRF journal should be retained"); + + let manifest_path = disk + .get_object_path(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0) + .expect("MRF manifest path should resolve"); + let parent = manifest_path.parent().expect("MRF manifest path should have a parent"); + os::fsync_dir_recorder::set_failure(parent, ErrorKind::StorageFull); + + let err = disk + .compare_and_update_file(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0, Some(committed_manifest.clone()), None) + .await + .expect_err("storage-full fsync failure must fail the MRF manifest cleanup delete"); + assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::StorageFull)); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0) + .await + .expect("committed MRF manifest should be restored after failed cleanup delete"), + committed_manifest + ); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("legacy MRF journal should remain readable after failed cleanup delete"), + legacy_journal + ); + } + #[cfg(unix)] #[tokio::test] async fn conditional_file_update_dir_fsync_failure_removes_new_file_without_anchor() { diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index ccb950c27..aa32f6fd7 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -279,11 +279,14 @@ impl LocalDisk { Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), None => None, }; + // Quota admission -> metadata RMW -> namespace/volume publication. + let metadata_lease = + os::acquire_metadata_mutation_lease(&self.get_object_path(dst_volume, dst_path)?, state.namespace_owner.take()).await; let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner( &self.root, dst_volume, &destination_object_path, - state.namespace_owner.take(), + Some(metadata_lease), ) .await; if let Some(claim) = quota_fence_claim { diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 1b8d23e58..a3c4af8d2 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -1277,6 +1277,25 @@ pub struct CheckPartsResp { pub struct UpdateMetadataOpts { pub no_persistence: bool, pub replace_user_metadata: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transition_reconcile: Option>, +} + +/// An exact-copy precondition for the single-version tier repair protocol. +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TransitionStateReconcileCondition { + pub expected_metadata_digest: String, + pub unchanged_metadata_digest: String, + pub target: rustfs_filemeta::TransitionStateReconcileTarget, + pub tier: String, + pub topology_generation: String, + pub verify_only: bool, + /// Local ownership is never accepted from the wire. A remote disk acquires + /// its own fleet and backend leases before entering the mutation domain. + #[serde(skip)] + pub(crate) authority: + Option>, } pub struct DiskLocation { diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 5f6335dbf..14a56b598 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -452,13 +452,13 @@ pub(crate) mod windows_rename_test_hooks { /// Test-only hooks into the destination-parent walk of rename preparation. /// -/// The prune race lives between two syscalls inside +/// Pruning and Windows sharing races live between syscalls inside /// [`mkdir_all_below_existing_base_std`], so only an injection at that exact /// point reproduces it deterministically. Hooks are keyed by the absolute path /// of the component just opened and queued per path: a retrying preparation /// visits the same component again, so a test models a pruner that keeps /// walking upward by queueing one hook per visit. -#[cfg(all(test, unix))] +#[cfg(all(test, any(unix, windows)))] pub(crate) mod prepare_rename_test_hooks { use super::*; @@ -1449,6 +1449,38 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc { lock } +static DISK_METADATA_MUTATION_LOCKS: LazyLock> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +/// Serializes the complete xl.meta read/modify/commit transaction. This domain +/// precedes namespace/volume publication locks, whose narrower syscall leases +/// may retain it after cancellation of the async caller. +pub(crate) struct MetadataMutationLease { + _guard: OwnedMutexGuard<()>, + _owner: Option>, +} + +pub(crate) async fn acquire_metadata_mutation_lease( + object: &Path, + owner: Option>, +) -> Arc { + let lock = { + let mut locks = DISK_METADATA_MUTATION_LOCKS.lock(); + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(object).and_then(Weak::upgrade) { + lock + } else { + let lock = Arc::new(AsyncMutex::new(())); + locks.insert(object.to_path_buf(), Arc::downgrade(&lock)); + lock + } + }; + Arc::new(MetadataMutationLease { + _guard: lock.lock_owned().await, + _owner: owner, + }) +} + /// Keeps a namespace transaction serialized even when its async waiter is /// cancelled while a blocking filesystem call is still running. pub(crate) struct NamespaceMutationLease { @@ -2112,6 +2144,41 @@ pub(crate) async fn rename_all_with_lease( Ok(()) } +/// Publish a conditional repair and sync its directory in one owned executor. +/// Cancellation cannot release its metadata/fleet/tier leases between rename +/// and fsync. The last authority check runs after destination preparation. +pub(crate) async fn rename_reconciled_metadata( + source: PathBuf, + destination: PathBuf, + base_dir: PathBuf, + publication_root: PublicationRoot, + owner: Option>, + authority: Arc, +) -> Result<()> { + let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await; + run_blocking_namespace_operation(lease, move || { + let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?; + #[cfg(all(any(test, feature = "test-util"), not(windows)))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination); + if !authority.is_current() { + return Err(io::Error::new(io::ErrorKind::WouldBlock, "transition reconciliation authority expired")); + } + rename_prepared(&source, &destination, &preparation)?; + if let Some(parent) = destination.parent() { + fsync_dir_std(parent)?; + } + Ok(()) + }) + .await + .map_err(|err| { + if err.kind() == io::ErrorKind::WouldBlock { + DiskError::OutdatedXLMeta + } else { + to_file_error(err).into() + } + }) +} + #[cfg(windows)] #[tracing::instrument(level = "debug", skip_all)] pub(crate) async fn rename_all_with_commit_guard( @@ -4208,11 +4275,14 @@ pub(crate) fn mkdir_all_below_existing_base_std( let mut handles = Vec::with_capacity(capacity); handles.push(publication_root.directory.clone()); let mut guard = ExistingBaseDirectoryGuard::new(handles); - for component in base_relative.components() { + let mut components = base_relative + .components() + .map(|component| (component, FILE_OPEN)) + .chain(relative.components().map(|component| (component, FILE_OPEN_IF))) + .filter(|(component, _)| !matches!(component, Component::CurDir)) + .peekable(); + while let Some((component, disposition)) = components.next() { let Component::Normal(component) = component else { - if matches!(component, Component::CurDir) { - continue; - } return Err(io::Error::new( io::ErrorKind::InvalidInput, "rename base directory contains an invalid path component", @@ -4222,40 +4292,22 @@ pub(crate) fn mkdir_all_below_existing_base_std( .handles .last() .ok_or_else(|| io::Error::other("Windows publication root guard is empty"))?; - let child = open_windows_directory_component(parent, component, FILE_OPEN)?; - guard.handles.push(child); - } - for component in relative.components() { - let Component::Normal(component) = component else { - continue; + // The kernel opens the final parent for write during a relative + // rename. Share writes from its first open: a temporary read-only + // share would block another rename into the same trash directory. + // Ancestors stay strict and no handle shares delete access, keeping + // every retained directory identity pinned. + let share_access = if components.peek().is_none() { + FILE_SHARE_READ | FILE_SHARE_WRITE + } else { + FILE_SHARE_READ }; - let parent = guard - .handles - .last() - .ok_or_else(|| io::Error::other("Windows base directory guard is empty"))?; - let child = open_windows_directory_component(parent, component, FILE_OPEN_IF)?; + let child = open_windows_relative_directory_component(parent, component, disposition, share_access)?; guard.handles.push(child); - } - - // Windows resolves a handle-relative rename by opening the target for - // write. Keep every ancestor strict, but let that internal open share - // the final parent. Delete sharing remains omitted, so the retained - // directory entry cannot be renamed or removed during publication. - if guard.handles.len() > 1 { - let component = dir_path - .file_name() - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination parent must have a name"))?; - let parent_index = guard.handles.len() - 2; - let parent = guard - .handles - .get(parent_index) - .ok_or_else(|| io::Error::other("Windows destination guard lost its parent handle"))?; - let rename_parent = - open_windows_relative_directory_component(parent, component, FILE_OPEN, FILE_SHARE_READ | FILE_SHARE_WRITE)?; - *guard - .handles - .last_mut() - .ok_or_else(|| io::Error::other("Windows destination guard is empty"))? = rename_parent; + #[cfg(test)] + if components.peek().is_none() { + prepare_rename_test_hooks::run_after_component_opened(dir_path); + } } Ok(guard) @@ -5446,6 +5498,111 @@ mod tests { assert_eq!(std::fs::read(dst).expect("read committed metadata"), b"metadata"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn rename_all_concurrent_trash_renames_remove_every_rollback_directory() { + let temp_dir = tempdir().expect("create temp dir"); + let trash = temp_dir.path().join(".rustfs.sys/tmp/.trash"); + std::fs::create_dir_all(&trash).expect("create trash directory"); + let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root"); + let sources: Vec<_> = (0..16) + .map(|index| { + let source = temp_dir + .path() + .join(format!("bucket/{index}.mp4")) + .join(uuid::Uuid::new_v4().to_string()); + std::fs::create_dir_all(&source).expect("create rollback directory"); + std::fs::write(source.join("xl.meta.bkp"), b"rollback metadata").expect("write metadata backup"); + source + }) + .collect(); + + let results = futures::future::join_all(sources.iter().enumerate().map(|(index, source)| { + super::rename_all_ignore_missing_source(source, trash.join(index.to_string()), &trash, &publication_root) + })) + .await; + + for (index, (source, result)) in sources.iter().zip(results).enumerate() { + result.expect("concurrent rollback cleanup must reach the shared trash directory"); + assert!(!source.exists(), "rollback cleanup must not leave a directory in the bucket"); + assert_eq!( + std::fs::read(trash.join(index.to_string()).join("xl.meta.bkp")).expect("read moved metadata backup"), + b"rollback metadata", + "trash staging must retain the complete backup" + ); + } + } + + #[cfg(windows)] + #[test] + fn windows_trash_rename_succeeds_during_concurrent_parent_preparation() { + use std::os::windows::fs::OpenOptionsExt; + use std::sync::atomic::{AtomicBool, Ordering}; + use windows_sys::Win32::{ + Foundation::{ERROR_SHARING_VIOLATION, GENERIC_WRITE}, + Storage::FileSystem::{ + DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ, + FILE_SHARE_WRITE, + }, + }; + + // Exercise the final parent both in the existing base walk and in the + // creatable suffix walk. Neither may briefly deny write sharing. + for nested_parent in [false, true] { + let temp_dir = tempdir().expect("create temp dir"); + let tmp = temp_dir.path().join(".rustfs.sys/tmp"); + let trash = tmp.join(".trash"); + std::fs::create_dir_all(&trash).expect("create trash directory"); + let base = if nested_parent { &tmp } else { &trash }; + let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root"); + let sources = ["first", "second"].map(|name| { + let source = temp_dir + .path() + .join("bucket") + .join(name) + .join(uuid::Uuid::new_v4().to_string()); + std::fs::create_dir_all(&source).expect("create rollback directory"); + std::fs::write(source.join("xl.meta.bkp"), name.as_bytes()).expect("write metadata backup"); + source + }); + let destinations = [trash.join("first"), trash.join("second")]; + let first_preparation = prepare_rename_with_retry(&sources[0], &destinations[0], base, &publication_root) + .expect("prepare the first trash rename"); + let first_source = sources[0].clone(); + let first_destination = destinations[0].clone(); + let interleaved = Arc::new(AtomicBool::new(false)); + let interleaved_hook = Arc::clone(&interleaved); + prepare_rename_test_hooks::queue_after_component_opened(&trash, move || { + interleaved_hook.store(true, Ordering::Release); + rename_prepared(&first_source, &first_destination, &first_preparation) + .expect("another preparation's first parent handle must allow the pending trash rename"); + }); + + let second_preparation = prepare_rename_with_retry(&sources[1], &destinations[1], base, &publication_root) + .expect("prepare the second trash rename"); + assert!(interleaved.load(Ordering::Acquire), "the competing parent-open window must be exercised"); + for (path, access) in [(&tmp, GENERIC_WRITE), (&trash, DELETE)] { + let err = std::fs::OpenOptions::new() + .access_mode(access) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) + .expect_err("ancestor writes and final-parent deletion must remain excluded"); + assert_eq!( + err.raw_os_error(), + Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32")) + ); + } + rename_prepared(&sources[1], &destinations[1], &second_preparation).expect("publish the second trash rename"); + for (index, payload) in [b"first".as_slice(), b"second".as_slice()].into_iter().enumerate() { + assert!(!sources[index].exists(), "both rollback directories must leave the bucket"); + assert_eq!( + std::fs::read(destinations[index].join("xl.meta.bkp")).expect("read the staged backup"), + payload + ); + } + } + } + #[cfg(windows)] #[tokio::test] async fn windows_rename_all_supports_same_parent_publication() { diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index 5b9bada4c..54150d6ae 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -14,7 +14,7 @@ use std::{ collections::{HashMap, HashSet}, - sync::{Arc, OnceLock}, + sync::{Arc, LazyLock, OnceLock, RwLock as StdRwLock}, time::SystemTime, }; @@ -57,6 +57,13 @@ use uuid::Uuid; const TEST_RPC_SECRET: &str = "test-rpc-secret"; pub(crate) type WorkloadSnapshotProviderRef = Arc; +pub type ScannerDirtyUsageMutationObserver = Arc; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScannerDirtyUsageMutationSource { + Replication, + TierExpiration, +} #[derive(Clone, Default)] pub(crate) struct LockRegistry { @@ -88,6 +95,8 @@ impl LockRegistry { } static WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER: OnceLock = OnceLock::new(); +static SCANNER_DIRTY_USAGE_MUTATION_OBSERVER: LazyLock>> = + LazyLock::new(|| StdRwLock::new(None)); pub(crate) fn set_workload_admission_snapshot_provider( provider: WorkloadSnapshotProviderRef, @@ -99,6 +108,28 @@ pub(crate) fn workload_admission_snapshot_provider() -> Option, +) -> Option { + let mut slot = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + std::mem::replace(&mut *slot, observer) +} + +pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) { + if bucket.is_empty() || object.is_empty() { + return; + } + let observer = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + if let Some(observer) = observer { + observer(bucket, object, source); + } +} + pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_error: &'static str) { global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error); } @@ -580,12 +611,16 @@ pub(crate) async fn init_tier_config_mgr(store: Arc) -> Result<()> { #[cfg(test)] mod tests { use super::{ - LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, reconcile_local_disk_ids, - replace_local_disk_id, set_local_node_name, + LockRegistry, ScannerDirtyUsageMutationSource, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, + notify_scanner_dirty_usage_mutation, reconcile_local_disk_ids, replace_local_disk_id, set_local_node_name, + set_scanner_dirty_usage_mutation_observer, }; use crate::disk::endpoint::Endpoint; use rustfs_lock::{LocalClient, LockClient}; - use std::{collections::HashMap, sync::Arc}; + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; use uuid::Uuid; fn url_endpoint(raw: &str) -> Endpoint { @@ -620,6 +655,42 @@ mod tests { assert!(Arc::ptr_eq(&clients[1], &client_b)); } + #[test] + #[serial_test::serial(scanner_dirty_usage_mutation_observer)] + fn scanner_dirty_usage_mutation_observer_filters_empty_identity_and_preserves_source() { + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_clone = Arc::clone(&observed); + let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| { + observed_clone.lock().expect("observer lock should not be poisoned").push(( + bucket.to_string(), + object.to_string(), + source, + )); + }))); + + notify_scanner_dirty_usage_mutation("photos", "2026/image.jpg", ScannerDirtyUsageMutationSource::Replication); + notify_scanner_dirty_usage_mutation("", "2026/empty-bucket.jpg", ScannerDirtyUsageMutationSource::TierExpiration); + notify_scanner_dirty_usage_mutation("photos", "", ScannerDirtyUsageMutationSource::TierExpiration); + notify_scanner_dirty_usage_mutation("archive", "expired.bin", ScannerDirtyUsageMutationSource::TierExpiration); + set_scanner_dirty_usage_mutation_observer(previous); + + assert_eq!( + *observed.lock().expect("observer lock should not be poisoned"), + vec![ + ( + "photos".to_string(), + "2026/image.jpg".to_string(), + ScannerDirtyUsageMutationSource::Replication + ), + ( + "archive".to_string(), + "expired.bin".to_string(), + ScannerDirtyUsageMutationSource::TierExpiration + ), + ] + ); + } + #[tokio::test] #[serial_test::serial] async fn local_node_name_round_trips_through_common_runtime_helper() { diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index ee6bc7dd7..23738641e 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -67,12 +67,9 @@ const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4; // Keep this synchronized with the version served by node_service. Including // the local member in the minimum prevents an older coordinator from // self-authorizing a policy implemented only by newer remote peers. -const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4; -/// Version 5 is reserved for a fleet whose every metadata writer preserves -/// explicit transition version state and destination identity, and implements -/// conditional per-generation `xl.meta` writes with strong readback. The node -/// service must not advertise this version until the conditional writer from -/// rustfs/backlog#684 is available. +const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 5; +/// Version 5 preserves explicit transition state/destination bindings and +/// supports exact-generation metadata repair with strong all-copy readback. const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5; fn resolve_admin_peer_probe_timeout_secs(configured: Option) -> u64 { @@ -568,6 +565,10 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint) } +pub(crate) fn cross_pool_fence_topology_generation(proof: &CrossPoolFenceFleetProofToken) -> String { + stable_tier_delete_journal_topology_generation(&proof.0.topology_fingerprint) +} + /// Acquire one non-cloneable authority that must span the complete reconcile /// effect window, including its final strong readback. pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option { @@ -603,6 +604,10 @@ fn acquire_legacy_transition_state_reconcile_fleet_proof_from( } async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option> { + #[cfg(all(test, feature = "test-util"))] + if let Ok(observation) = LEGACY_RECONCILE_TEST_OBSERVATION.try_with(Clone::clone) { + return Some(observation); + } let notification_sys = get_global_notification_sys()?; let (peer_epochs, minimum_version) = timeout( REMOTE_VERSION_STATE_PROBE_TIMEOUT, @@ -615,6 +620,34 @@ async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str reconcile_result.ok() } +#[cfg(all(test, feature = "test-util"))] +tokio::task_local! { + static LEGACY_RECONCILE_TEST_OBSERVATION: BTreeMap; +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) async fn with_legacy_transition_state_fleet_proof_for_test(future: F) -> F::Output { + struct Revoke; + impl Drop for Revoke { + fn drop(&mut self) { + revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot()); + } + } + let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().expect("test store topology"); + assert!( + publish_fleet_capability_probe_result( + legacy_transition_state_reconcile_fleet_proof_slot(), + topology, + Ok(BTreeMap::new()), + Instant::now(), + ) + .is_none() + ); + let _revoke = Revoke; + let _remote_version = install_current_remote_version_state_fleet_proof_for_test(); + LEGACY_RECONCILE_TEST_OBSERVATION.scope(BTreeMap::new(), future).await +} + /// Revalidate the exact fleet generation captured by a reconcile token with a /// fresh synchronous observation. Callers must await this before each /// conditional metadata write and after the final strong readback. @@ -633,6 +666,18 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches( .await } +/// Final local check in the disk publication executor. The corresponding +/// counted permit remains owned until the filesystem operation has drained. +pub(crate) fn legacy_transition_state_reconcile_fleet_proof_current( + proof: &LegacyTransitionStateReconcileFleetProofToken, +) -> bool { + let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false }; + let state = legacy_transition_state_reconcile_fleet_proof_slot() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, topology, Instant::now()) +} + pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option { let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?; let proof = { @@ -1120,6 +1165,14 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp RemoteVersionStateFleetProofGuard } +#[cfg(all(test, feature = "test-util"))] +pub(crate) fn install_current_remote_version_state_fleet_proof_for_test() -> RemoteVersionStateFleetProofGuard { + let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY + .get() + .expect("the test store must bind its fleet topology before installing a writer proof"); + install_remote_version_state_fleet_proof_for_test(topology) +} + #[cfg(all(test, feature = "test-util"))] pub(crate) struct TransitionTransactionCompactionFleetProofGuard; @@ -3914,15 +3967,11 @@ mod tests { assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence"); assert!(reconcile_v3.is_err()); - let (generic_v4, journal_v4, decommission_v4, reconcile_v4) = - cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION); + let (generic_v4, journal_v4, decommission_v4, reconcile_v4) = cross_pool_fence_policy_results(peers.clone(), 4); assert!(generic_v4.is_ok()); assert!(journal_v4.is_ok()); assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations"); - assert!( - reconcile_v4.is_err(), - "the current local policy lacks the conditional xl.meta writer required by reconcile" - ); + assert!(reconcile_v4.is_err(), "v4 does not support conditional transition metadata writes"); let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5); assert!(generic_v5.is_ok()); @@ -4600,7 +4649,7 @@ mod tests { } #[tokio::test] - async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() { + async fn legacy_transition_state_reconcile_single_node_advertises_conditional_writer() { let notification_sys = NotificationSys { peer_clients: Vec::new(), all_peer_clients: vec![None], @@ -4616,8 +4665,8 @@ mod tests { assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION); let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version); assert!( - reconcile_result.is_err(), - "the current node must not self-authorize reconcile before the conditional writer lands" + reconcile_result.is_ok(), + "the current node implements the conditional writer and preserves repaired bindings" ); } diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index 09c6dee81..96e9b92fc 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -653,6 +653,26 @@ impl MockWarmBackend { #[async_trait] impl WarmBackend for MockWarmBackend { + async fn probe_legacy_metadata( + &self, + object: &str, + remote_version: Option<&str>, + ) -> Result { + use super::warm_backend::LegacyTransitionStateProbe as Probe; + let candidate = match remote_version { + Some(version) if !version.is_empty() => self.probe_transition_version(object, version).await?, + _ => self.probe_transition_candidate(object).await?, + }; + Ok(match candidate { + TransitionCandidateProbe::Missing => Probe::Missing, + TransitionCandidateProbe::UnversionedPresent => Probe::UnversionedPresent, + TransitionCandidateProbe::VersionedPresent(version) if version == "null" => Probe::SuspendedNullPresent, + TransitionCandidateProbe::VersionedPresent(version) => Probe::VersionedPresent(version), + TransitionCandidateProbe::Ambiguous => Probe::Ambiguous, + TransitionCandidateProbe::Unsupported => Probe::Unsupported, + }) + } + fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> { if remote_version_id.is_empty() { return Ok(()); @@ -874,8 +894,9 @@ pub async fn register_mock_tier_backend(handle: &Arc>, tie ..Default::default() }, ); - tier_config_mgr - .install_test_driver(tier_name, Box::new(backend)) + drop(tier_config_mgr); + TierConfigMgr::install_test_driver_in(handle, tier_name, Box::new(backend)) + .await .expect("mock tier driver should install"); } diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 5e67673e9..8d80f37e9 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -2322,6 +2322,14 @@ struct SharedWarmBackendProxy(SharedWarmBackend); #[async_trait::async_trait] impl WarmBackend for SharedWarmBackendProxy { + async fn probe_legacy_metadata( + &self, + object: &str, + remote_version: Option<&str>, + ) -> io::Result { + self.0.probe_legacy_metadata(object, remote_version).await + } + async fn validate(&self) -> io::Result<()> { self.0.validate().await } @@ -2490,6 +2498,27 @@ impl TierOperationLease { self.inner.driver.probe_transition_version(object, remote_version_id).await } + pub(crate) async fn probe_legacy_transition_state( + &self, + object: &str, + remote_version: Option<&str>, + ) -> io::Result { + let Some(reconciler) = self + .inner + .reconciler + .get_or_try_init(|| async { + crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config) + .await + .map(|reconciler| reconciler.map(Arc::from)) + }) + .await + .map_err(|err| io::Error::other(err.message))? + else { + return self.inner.driver.probe_legacy_metadata(object, remote_version).await; + }; + reconciler.probe_legacy_transition_state(object, remote_version).await + } + pub(crate) fn is_current_generation(&self) -> bool { lock_unpoisoned(&self.runtime) .generations @@ -6013,6 +6042,19 @@ impl TierConfigMgr { Ok(()) } + #[cfg(any(test, feature = "test-util"))] + pub(crate) async fn install_test_driver_in( + handle: &Arc>, + tier_name: &str, + driver: WarmBackendImpl, + ) -> std::result::Result<(), AdminError> { + let mut manager = handle.write().await; + // Register the generation runtime before installing the mock so its + // explicit lack of a network reconciler survives the first lease. + tier_driver_runtime(handle, &manager); + manager.install_test_driver(tier_name, driver) + } + #[cfg(any(test, feature = "test-util"))] pub(crate) fn install_test_driver( &mut self, @@ -6391,9 +6433,48 @@ impl TierConfigMgr { } pub(crate) async fn refresh_tier_config_handle(handle: Arc>, api: Arc) { - Self::refresh_tier_config_handle_with(handle, api).await; + Self::refresh_tier_config_handle_with_weak(handle, Arc::downgrade(&api)).await; } + async fn refresh_tier_config_handle_with_weak(handle: Arc>, api: Weak) { + // The periodic refresh remains the recovery fallback; committed mutations + // notify this worker so a successful peer commit converges immediately. + let mutation_refresh = Self::mutation_refresh_notifier(&handle).await; + let r = rand::rng().random_range(0.0..1.0); + let rand_interval = || Duration::from_secs((r * 60_f64).round() as u64); + + let refresh_interval = TIER_CFG_REFRESH + rand_interval(); + let mut t = delayed_tier_refresh_interval(refresh_interval); + loop { + select! { + _ = t.tick() => { + let Some(api) = Weak::upgrade(&api) else { + return; + }; + if let Err(err) = Self::reload_handle_with(&handle, api).await { + warn!( + event = EVENT_TIER_CONFIG_REFRESH, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + trigger = "periodic", + result = "failed", + error = ?err, + "tier configuration refresh" + ); + } + } + _ = mutation_refresh.notified() => { + let Some(api) = Weak::upgrade(&api) else { + return; + }; + Self::reload_after_committed_mutation(&handle, api).await; + } + } + t.reset(); + } + } + + #[allow(dead_code, reason = "used by focused tier refresh tests and non-ECStore generic harnesses")] pub(crate) async fn refresh_tier_config_handle_with(handle: Arc>, api: Arc) where S: EcstoreObjectIO @@ -17498,6 +17579,66 @@ mod tests { assert!(current.tiers.contains_key("COLD-B")); } + async fn wait_for_reference_proof_barrier( + barrier: &TierDriverBuildBarrier, + update: &mut tokio::task::JoinHandle>, + ) -> std::result::Result<(), String> { + tokio::select! { + biased; + result = &mut *update => Err(format!("tier update exited before the reference proof barrier: {result:?}")), + () = barrier.arrived.notified() => Ok(()), + () = tokio::time::sleep(Duration::from_secs(30)) => { + // Aborting the caller does not stop its owned mutation task. + // Let a late arrival pass the test-only barrier. + barrier.release.add_permits(1); + update.abort(); + Err("timed out waiting for the reference proof barrier".to_string()) + } + } + } + + #[tokio::test] + #[serial_test::serial] + async fn reference_proof_barrier_reports_update_failure_before_arrival() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("early update failure fixture should persist"); + let barrier = tier_reference_proof_test_barrier(); + let scoped_barrier = barrier.clone(); + let factory: TierDriverTestFactory = + Arc::new(|_| Err(AdminError::msg("injected driver initialization failure before reference proof"))); + let mut update = tokio::spawn(async move { + TIER_REFERENCE_PROOF_TEST_BARRIER + .scope( + scoped_barrier, + TIER_DRIVER_TEST_FACTORY.scope( + factory, + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::update_candidate_with_config_lock( + &manager, + store, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + ), + ), + ), + ) + .await + }); + + let err = tokio::time::timeout(Duration::from_secs(5), wait_for_reference_proof_barrier(&barrier, &mut update)) + .await + .expect("an early update failure should be observed without waiting for the barrier deadline") + .expect_err("a failed update cannot reach the reference proof barrier"); + assert!(err.contains("Mutation"), "{err}"); + assert!(err.contains("injected driver initialization failure before reference proof"), "{err}"); + } + #[tokio::test] #[serial_test::serial] async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() { @@ -17518,7 +17659,7 @@ mod tests { let scoped_barrier = barrier.clone(); let update_manager = manager.clone(); let update_store = store.clone(); - let update = tokio::spawn(async move { + let mut update = tokio::spawn(async move { TIER_REFERENCE_PROOF_TEST_BARRIER .scope( scoped_barrier, @@ -17533,7 +17674,9 @@ mod tests { ) .await }); - barrier.arrived.notified().await; + wait_for_reference_proof_barrier(&barrier, &mut update) + .await + .expect("tier update should reach the reference proof barrier"); let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237)); TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated) @@ -17541,8 +17684,9 @@ mod tests { .expect("an unrelated prepared fence should advance the runtime revision"); barrier.release.add_permits(1); - let err = update + let err = tokio::time::timeout(Duration::from_secs(30), update) .await + .expect("tier update should finish after the reference proof barrier releases") .expect("tier update task should join") .expect_err("a reference proof cannot authorize publication across a fence revision change"); let TierConfigUpdateError::Publish(err) = err else { diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index fe4b5e118..8e7a4c8f4 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -89,6 +89,18 @@ pub enum TransitionCandidateProbe { Unsupported, } +/// Live evidence for repairing legacy metadata. Ordinary candidate GETs do not +/// establish the bucket's versioning model and cannot supply this authority. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum LegacyTransitionStateProbe { + Missing, + UnversionedPresent, + SuspendedNullPresent, + VersionedPresent(String), + Ambiguous, + Unsupported, +} + #[derive(Clone, Copy)] pub(crate) struct TransitionCandidateIdentity { pub transaction_id: uuid::Uuid, @@ -97,6 +109,14 @@ pub(crate) struct TransitionCandidateIdentity { #[async_trait::async_trait] pub(crate) trait TransitionCandidateReconciler { + async fn probe_legacy_transition_state( + &self, + _object: &str, + _remote_version: Option<&str>, + ) -> Result { + Ok(LegacyTransitionStateProbe::Unsupported) + } + async fn probe_transition_candidate_for( &self, object: &str, @@ -106,6 +126,14 @@ pub(crate) trait TransitionCandidateReconciler { #[async_trait::async_trait] pub trait WarmBackend { + async fn probe_legacy_metadata( + &self, + _object: &str, + _remote_version: Option<&str>, + ) -> Result { + Ok(LegacyTransitionStateProbe::Unsupported) + } + async fn validate(&self) -> Result<(), std::io::Error> { Ok(()) } @@ -448,6 +476,18 @@ impl MeteredWarmBackend { #[async_trait::async_trait] impl WarmBackend for MeteredWarmBackend { + async fn probe_legacy_metadata( + &self, + object: &str, + remote_version: Option<&str>, + ) -> Result { + let result = self.inner.probe_legacy_metadata(object, remote_version).await; + if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) { + return result; + } + Self::record(TierRequestOperation::Probe, result) + } + /// Delegated without a counter: only one backend issues a remote request /// here, and every other one takes the trait default, so a `validate` /// counter would mostly record requests that never happened. @@ -524,6 +564,18 @@ struct MeteredTransitionCandidateReconciler { #[async_trait::async_trait] impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler { + async fn probe_legacy_transition_state( + &self, + object: &str, + remote_version: Option<&str>, + ) -> Result { + let result = self.inner.probe_legacy_transition_state(object, remote_version).await; + if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) { + return result; + } + MeteredWarmBackend::record(TierRequestOperation::Probe, result) + } + async fn probe_transition_candidate_for( &self, object: &str, diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index 6baef149b..34657ef9a 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -101,6 +101,19 @@ impl WarmBackend for WarmBackendMinIO { #[async_trait::async_trait] impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO { + async fn probe_legacy_transition_state( + &self, + object: &str, + remote_version: Option<&str>, + ) -> Result { + crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state( + &self.0, + object, + remote_version, + ) + .await + } + async fn probe_transition_candidate_for( &self, object: &str, diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 0bb19bcdf..9c1934a6d 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -146,6 +146,19 @@ impl WarmBackend for WarmBackendRustFS { #[async_trait::async_trait] impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS { + async fn probe_legacy_transition_state( + &self, + object: &str, + remote_version: Option<&str>, + ) -> Result { + crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state( + &self.0, + object, + remote_version, + ) + .await + } + async fn probe_transition_candidate_for( &self, object: &str, diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 8c1d6ef9b..4e790b560 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -376,7 +376,6 @@ struct TransitionCandidateVersions { } impl TransitionCandidateVersions { - #[cfg(test)] fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) { for version in versions.versions.iter().filter(|version| version.key == remote_object) { if self.version_id.is_some() { @@ -512,6 +511,20 @@ mod tests { } async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle>)> { + scripted_probe_fixture([ + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx", + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\nNoSuchObjectmissing", + "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\nAccessDenieddenied", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\nInvalidRangeempty version", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\nNoSuchVersionmissing", + "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", + ].into_iter().map(str::to_owned).collect()).await + } + + async fn scripted_probe_fixture(responses: Vec) -> Option<(WarmBackendS3, tokio::task::JoinHandle>)> { let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { Ok(listener) => listener, Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, @@ -522,17 +535,6 @@ mod tests { .expect("listener local address should be available") .to_string(); let fixture = tokio::spawn(async move { - let responses = [ - "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx", - "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx", - "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", - "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\nNoSuchObjectmissing", - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\nAccessDenieddenied", - "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", - "HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\nInvalidRangeempty version", - "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\nNoSuchVersionmissing", - "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\nNoSuchKeymissing", - ]; let mut requests = Vec::new(); for response in responses { let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET"); @@ -673,6 +675,117 @@ mod tests { assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version")); } + fn legacy_probe_xml_response(body: &str) -> String { + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + } + + fn legacy_probe_versioning_response(status: &str) -> String { + let state = if status.is_empty() { + String::new() + } else { + format!("{status}") + }; + legacy_probe_xml_response(&format!( + "{state}" + )) + } + + fn legacy_probe_versions_response(versions: &[&str]) -> String { + let versions = versions.iter().map(|version| format!( + "archive/object{version}true2026-09-01T00:00:00Z\"legacy-etag\"7STANDARD" + )).collect::(); + legacy_probe_xml_response(&format!( + "bucketarchive/object1000false{versions}" + )) + } + + #[tokio::test] + async fn legacy_transition_state_probe_verifies_disabled_suspended_and_enabled_responses() { + use super::super::warm_backend::LegacyTransitionStateProbe as Probe; + for (initial, confirmed, version, expected) in [ + ("", "", "null", Probe::UnversionedPresent), + ("Suspended", "Suspended", "null", Probe::SuspendedNullPresent), + ("Enabled", "Enabled", "version-a", Probe::VersionedPresent("version-a".to_string())), + ("Enabled", "Enabled", "null", Probe::SuspendedNullPresent), + ("", "", "unexpected-version", Probe::Ambiguous), + ("Suspended", "Enabled", "null", Probe::Ambiguous), + ] { + let responses = vec![ + legacy_probe_versioning_response(initial), + legacy_probe_versions_response(&[version]), + legacy_probe_versioning_response(confirmed), + ]; + let (backend, fixture) = scripted_probe_fixture(responses) + .await + .expect("legacy probe loopback fixture"); + let result = + tokio::time::timeout(Duration::from_secs(10), backend.probe_legacy_transition_state("archive/object", None)) + .await + .expect("legacy probe must finish") + .expect("legacy probe should decode provider XML"); + assert_eq!(result, expected, "initial={initial} confirmed={confirmed} version={version}"); + let requests = fixture.await.expect("legacy probe fixture should finish"); + assert_eq!(requests.len(), 3); + assert!(requests.iter().all(|request| request.starts_with("GET "))); + assert!(requests[0].lines().next().expect("request line").contains("versioning")); + assert!(requests[1].lines().next().expect("request line").contains("versions")); + assert!(requests[2].lines().next().expect("request line").contains("versioning")); + } + } + + #[tokio::test] + async fn legacy_transition_state_probe_preserves_historical_exact_version() { + use super::super::warm_backend::LegacyTransitionStateProbe as Probe; + let responses = vec![ + legacy_probe_versioning_response("Enabled"), + "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: historical-version\r\nConnection: close\r\n\r\nx".to_string(), + legacy_probe_versioning_response("Enabled"), + ]; + let (backend, fixture) = scripted_probe_fixture(responses).await.expect("exact legacy probe fixture"); + assert_eq!( + backend + .probe_legacy_transition_state("archive/object", Some("historical-version")) + .await + .expect("exact version proof"), + Probe::VersionedPresent("historical-version".to_string()) + ); + let requests = fixture.await.expect("exact probe fixture should finish"); + assert!( + requests[1] + .lines() + .next() + .expect("request line") + .contains("versionId=historical-version") + ); + assert!(requests[1].to_ascii_lowercase().contains("range: bytes=0-0")); + assert!(requests.iter().all(|request| request.starts_with("GET "))); + } + + #[tokio::test] + async fn legacy_transition_state_probe_retains_multiple_candidates() { + use super::super::warm_backend::LegacyTransitionStateProbe as Probe; + let responses = vec![ + legacy_probe_versioning_response("Enabled"), + legacy_probe_versions_response(&["version-a", "version-b"]), + ]; + let (backend, fixture) = scripted_probe_fixture(responses) + .await + .expect("ambiguous legacy probe fixture"); + assert_eq!( + backend + .probe_legacy_transition_state("archive/object", None) + .await + .expect("ambiguous proof"), + Probe::Ambiguous + ); + let requests = fixture.await.expect("ambiguous fixture should finish"); + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| request.starts_with("GET "))); + } + fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult { ListVersionsResult { versions: versions @@ -921,6 +1034,56 @@ impl WarmBackend for WarmBackendS3 { #[async_trait::async_trait] impl TransitionCandidateReconciler for WarmBackendS3 { + async fn probe_legacy_transition_state( + &self, + object: &str, + remote_version: Option<&str>, + ) -> Result { + use super::warm_backend::LegacyTransitionStateProbe as Probe; + + let initial_versioning = self.remote_bucket_versioning().await?; + let candidate = if let Some(version) = remote_version.filter(|version| !version.is_empty()) { + validate_remote_version_id(version)?; + match self.probe_transition_version(object, version).await? { + TransitionCandidateProbe::VersionedPresent(actual) if actual == version => Some(actual), + TransitionCandidateProbe::Missing => return Ok(Probe::Missing), + _ => return Ok(Probe::Ambiguous), + } + } else { + let remote_object = self.get_dest(object); + let mut opts = ListObjectsOptions::default(); + opts.set("prefix", &remote_object); + opts.set("max-keys", "1000"); + let mut key_marker = String::new(); + let mut version_marker = String::new(); + let mut candidates = TransitionCandidateVersions::default(); + let mut complete = false; + // This is one synchronous record inspection, not an unbounded + // remote history scan. The caller also bounds the whole probe. + for _ in 0..128 { + let page = self + .client + .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_marker, "") + .await?; + candidates.extend(&remote_object, &page); + if candidates.ambiguous { + return Ok(Probe::Ambiguous); + } + if !page.is_truncated { + complete = true; + break; + } + advance_version_markers(&mut key_marker, &mut version_marker, &page)?; + } + if !complete { + return Ok(Probe::Ambiguous); + } + candidates.version_id + }; + let confirmed_versioning = self.remote_bucket_versioning().await?; + classify_legacy_transition_state(candidate.as_deref(), initial_versioning, confirmed_versioning) + } + async fn probe_transition_candidate_for( &self, object: &str, @@ -931,3 +1094,32 @@ impl TransitionCandidateReconciler for WarmBackendS3 { .await } } + +fn classify_legacy_transition_state( + candidate: Option<&str>, + initial: RemoteBucketVersioning, + confirmed: RemoteBucketVersioning, +) -> Result { + use super::warm_backend::LegacyTransitionStateProbe as Probe; + if initial != confirmed { + return Ok(Probe::Ambiguous); + } + let Some(version) = candidate else { + return Ok(Probe::Missing); + }; + if !version.is_empty() { + validate_remote_version_id(version)?; + if uuid::Uuid::parse_str(version).is_ok_and(|id| id.is_nil()) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy tier probe returned a nil version identifier", + )); + } + } + Ok(match (confirmed, version) { + (RemoteBucketVersioning::Disabled, "" | "null") => Probe::UnversionedPresent, + (RemoteBucketVersioning::Disabled, _) | (_, "") => Probe::Ambiguous, + (_, "null") => Probe::SuspendedNullPresent, + (_, version) => Probe::VersionedPresent(version.to_string()), + }) +} diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 4dc17ad34..e21969917 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -51,12 +51,12 @@ use super::super::{ can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding, collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset, - inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled, - is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled, - is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled, - issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions, - object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs, - send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn, + inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled, + is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, + is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, + merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, + reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err, + try_read_inline_data_shards_direct, warn, }; #[cfg(test)] pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision; @@ -3086,21 +3086,61 @@ impl SetDisks { let read_quorum = disks.len().div_ceil(2).max(1); let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await; - if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { - let object_err = to_object_err(err.into(), vec![bucket, object]); - if is_err_object_not_found(&object_err) || is_err_version_not_found(&object_err) { - return Ok(None); - } - return Err(object_err); + if let Some(err) = errs + .iter() + .flatten() + .find(|err| !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound)) + { + return Err(to_object_err(err.clone().into(), vec![bucket, object])); + } + // A minority live owner must not disappear behind majority absence. + // Only explicit absence on every readable disk proves no ownership. + if raw_fileinfos.iter().all(Option::is_none) { + return Ok(None); } let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len()); + type TransitionCopy = (FileInfo, Option); + let mut transition_copies: std::collections::HashMap, Vec> = + std::collections::HashMap::new(); + let decode_error = |err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}")); for raw_fileinfo in raw_fileinfos.into_iter().flatten() { let meta = FileMeta::load(&raw_fileinfo.buf) .map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?; + let versions = meta.get_all_file_info_versions(bucket, object, true).map_err(decode_error)?; + for version in versions.versions.into_iter().chain(versions.free_versions) { + if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE { + continue; + } + let destination = + crate::services::tier::tier::tier_destination_id_from_metadata(&version.metadata).map_err(Error::other)?; + transition_copies + .entry(version.version_id.filter(|id| !id.is_nil())) + .or_default() + .push((version, destination)); + } shallow_versions.push(meta.versions); } + // Exact cleanup/recovery reads must not select a repaired majority + // while another physical copy still carries legacy absence. Missing + // copies permit deletion retries; an unreadable disk proves nothing. + for copies in transition_copies + .values() + .filter(|copies| copies.iter().any(|(_, destination)| destination.is_some())) + { + let (first, destination) = &copies[0]; + if copies.iter().any(|(copy, identity)| { + identity != destination + || copy.transition_version_state != first.transition_version_state + || copy.transition_version != first.transition_version + || copy.transition_tier != first.transition_tier + || copy.transitioned_objname != first.transitioned_objname + }) { + return Err(Error::other("exact transition metadata has not converged across physical copies")); + } + } + if shallow_versions.len() < read_quorum { return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object])); } @@ -3117,7 +3157,7 @@ impl SetDisks { ..Default::default() } .get_all_file_info_versions(bucket, object, true) - .map_err(|err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}")))?; + .map_err(decode_error)?; for file_info in file_info_versions .versions @@ -5815,17 +5855,17 @@ impl SetDisks { } if let Some(disk) = disks[i].as_ref() { - let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]); + // A failed version-only copy owns only its new version. + // Removing the whole xl.meta would also erase existing + // versions and any concurrently reconciled tier binding. + let mut rollback = FileInfo { + version_id: files[i].version_id, + ..Default::default() + }; + rollback.set_skip_tier_free_version(); revert_futures.push(async move { if let Err(err) = disk - .delete( - bucket, - &path, - DeleteOptions { - recursive: true, - ..Default::default() - }, - ) + .delete_version(bucket, prefix, rollback, false, DeleteOptions::default()) .await { warn!("write meta revert err {:?}", err); @@ -12092,7 +12132,9 @@ mod tests { let bucket = "write-unique-bucket"; let object = "object"; let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await; - let files = vec![metadata_test_fileinfo(object), metadata_test_fileinfo(object)]; + let mut fi = metadata_test_fileinfo(object); + fi.mod_time = Some(OffsetDateTime::now_utc()); + let files = vec![fi.clone(), fi]; let result = SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &files, 2).await; @@ -12106,6 +12148,53 @@ mod tests { ); } + #[tokio::test] + async fn write_unique_file_info_rollback_preserves_existing_reconciled_version() { + let bucket = "write-unique-existing"; + let object = "object"; + let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await; + let mut original = metadata_test_fileinfo(object); + original.version_id = Some(Uuid::from_u128(1)); + original.data_dir = Some(Uuid::from_u128(3)); + original.mod_time = Some(OffsetDateTime::now_utc()); + original.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string(); + original.transition_tier = "WARM".to_string(); + original.transitioned_objname = "remote-original".to_string(); + original.transition_version_state = rustfs_filemeta::TransitionVersionState::KnownDisabled; + rustfs_utils::http::insert_str( + &mut original.metadata, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + "ab".repeat(32), + ); + disk.write_metadata(bucket, bucket, object, original.clone()) + .await + .expect("existing reconciled source"); + let raw = disk + .read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}")) + .await + .expect("original metadata"); + let before = FileMeta::load(&raw).unwrap().find_version(original.version_id).unwrap().1; + let mut added = original.clone(); + added.version_id = Some(Uuid::from_u128(2)); + let result = + SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &[added.clone(), added], 2) + .await; + assert!(result.is_err()); + let raw = disk + .read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}")) + .await + .expect("preserved xl.meta"); + let after = FileMeta::load(&raw).expect("preserved metadata"); + assert_eq!( + after + .find_version(original.version_id) + .expect("original source survives rollback") + .1, + before + ); + assert!(after.find_version(Some(Uuid::from_u128(2))).is_err()); + } + #[tokio::test] async fn update_object_meta_handles_empty_metadata_and_missing_quorum() { let set = io_primitives_test_set(vec![None, None], 1).await; diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 17129c9ae..f2cbafc55 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -3852,7 +3852,7 @@ pub struct SetDisks { pub default_parity_count: usize, pub set_index: usize, pub pool_index: usize, - /// Stable namespace shared by every object lock created for this set. + /// Stable namespace shared by every object lock created for this pool. set_lock_namespace: Arc, pub format: FormatV3, #[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")] @@ -3889,6 +3889,33 @@ pub struct SetDisks { >, } +/// Read every physical copy before selecting a version quorum. A minority +/// legacy record is still evidence and must not disappear behind a majority +/// not-found result. Only an explicit file/volume absence produces `None`; +/// an unreadable disk cannot prove that no conflicting copy exists. +pub(crate) async fn read_legacy_transition_state_metadata_copies( + set: &SetDisks, + bucket: &str, + object: &str, +) -> std::result::Result>>, DiskError> { + let disk_object = rustfs_utils::path::encode_dir_object(object); + let disks = set.get_disks_internal().await; + if disks.is_empty() { + return Err(DiskError::DiskNotFound); + } + + // Include inline bytes in the generation: a conditional repair preserves + // the entire xl.meta, including payloads belonging to other versions. + let (copies, errs) = SetDisks::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), true).await; + for err in errs.into_iter().flatten() { + if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) { + return Err(err); + } + } + + Ok(copies.into_iter().map(|copy| copy.map(|copy| copy.buf)).collect()) +} + // DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones // each endpoint's canonical Arc, so an exact Arc set identifies the lock domain. pub(crate) fn same_distributed_lock_domain(left: &[Arc], right: &[Arc]) -> bool { @@ -4237,7 +4264,7 @@ impl SetDisks { self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value } - async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) { + pub(crate) async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) { let hash = self.get_object_metadata_cache_hash(bucket, object); let hash_bytes = hash.to_le_bytes(); let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS); @@ -4464,7 +4491,7 @@ impl SetDisks { instance_ctx: Arc, ) -> Arc { let ctx = instance_ctx; - let set_lock_namespace: Arc = format!("set-{pool_index}-{set_index}").into(); + let set_lock_namespace: Arc = format!("pool-{pool_index}").into(); let shared_lockers = Arc::from(lockers.to_vec()); Arc::new(SetDisks { locker_owner, @@ -4578,7 +4605,9 @@ impl SetDisks { pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool { match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) { (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager), - (true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers), + (true, true) => { + self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers) + } _ => false, } } @@ -7096,7 +7125,7 @@ mod tests { ctx.update_erasure_type(SetupType::Erasure).await; let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await; - assert_eq!(&*set.set_lock_namespace, "set-0-0"); + assert_eq!(&*set.set_lock_namespace, "pool-0"); let before = Arc::strong_count(&set.set_lock_namespace); let lock = set .new_ns_lock("bucket", "object") @@ -8321,6 +8350,78 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn test_new_ns_lock_distributed_write_succeeds_with_three_lockers_one_offline() { + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + + let manager_a = Arc::new(rustfs_lock::GlobalLockManager::new()); + let manager_b = Arc::new(rustfs_lock::GlobalLockManager::new()); + let healthy_a: Arc = Arc::new(LocalClient::with_manager(manager_a)); + let healthy_b: Arc = Arc::new(LocalClient::with_manager(manager_b)); + let failing_client: Arc = Arc::new(FailingClient); + let set_disks = make_test_set_disks(vec![healthy_a, failing_client, healthy_b]).await; + + let guard = set_disks + .new_ns_lock("bucket", "object") + .await + .expect("namespace lock should be created") + .get_write_lock(Duration::from_millis(500)) + .await + .expect("two healthy lockers should satisfy the three-locker write quorum"); + + match guard { + NamespaceLockGuard::Standard(_) => {} + NamespaceLockGuard::Fast(_) => panic!("Expected distributed guard for dist-erasure"), + } + } + + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn namespace_lock_domain_includes_pool_namespace() { + let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; + + let first: Arc = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))); + let second: Arc = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))); + let lockers = vec![first, second]; + let same_pool_first_set = make_test_set_disks_with_ctx(lockers.clone(), bootstrap_ctx()).await; + let same_pool_second_set = SetDisks::new_with_instance_ctx( + "test-owner".to_string(), + Arc::new(RwLock::new(vec![None, None])), + 2, + 1, + 1, + 0, + same_pool_first_set.set_endpoints.clone(), + FormatV3::new(2, 2), + lockers.clone(), + bootstrap_ctx(), + ) + .await; + let other_pool_set = SetDisks::new_with_instance_ctx( + "test-owner".to_string(), + Arc::new(RwLock::new(vec![None, None])), + 2, + 1, + 0, + 1, + same_pool_first_set.set_endpoints.clone(), + FormatV3::new(1, 2), + lockers, + bootstrap_ctx(), + ) + .await; + + assert!( + same_pool_first_set.shares_namespace_lock_domain(&same_pool_second_set).await, + "sets in the same pool share the object namespace lock domain" + ); + assert!( + !same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await, + "different pool namespaces must not be deduplicated solely by identical clients" + ); + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn streaming_reader_holds_read_lock_until_eof() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index defd6315d..9cfcd4e4d 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -9209,6 +9209,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { let dest_obj = transaction.remote_object.clone(); let mut transition_meta = (*oi.user_defined).clone(); rustfs_utils::http::remove_str(&mut transition_meta, rustfs_utils::http::SUFFIX_PART_CHECKSUMS); + // The tier holds opaque stored bytes. Its metadata must not be treated + // as a second object header set: forwarding SSE intent or wrapped DEKs + // would request a second encryption pass and disclose local envelope + // material to the remote provider. + transition_meta.retain(|key, _| !rustfs_utils::http::is_replication_stripped_encryption_key(key)); transition_meta.insert("name".to_string(), object.to_string()); rustfs_utils::http::metadata_compat::insert_str( &mut transition_meta, @@ -9765,13 +9770,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { part_opts.part_number = Some(part_info.number); #[cfg(test)] fail_restore_multipart_at(RestoreMultipartFailurePoint::InvalidPartSize)?; - if part_info.actual_size <= 0 { - return Err(Error::other(format!("invalid multipart restore part size {}", part_info.actual_size))); + if part_info.size == 0 { + return Err(Error::other(format!("invalid multipart restore stored part size {}", part_info.size))); } + let stored_part_size = i64::try_from(part_info.size).map_err(|_| { + Error::other(format!("multipart restore stored part size exceeds i64: {}", part_info.size)) + })?; #[cfg(test)] fail_restore_multipart_at(RestoreMultipartFailurePoint::RangeOverflow)?; let part_end = part_offset - .checked_add(part_info.actual_size - 1) + .checked_add(stored_part_size - 1) .ok_or_else(|| Error::other("multipart restore part range overflow".to_string()))?; let rs = Some(HTTPRangeSpec { is_suffix_length: false, @@ -9799,13 +9807,19 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { #[cfg(test)] fail_restore_multipart_at(RestoreMultipartFailurePoint::HashReader)?; let hash_reader = - HashReader::from_stream(reader, part_info.actual_size, part_info.actual_size, None, None, false)?; + HashReader::from_stream(reader, stored_part_size, part_info.actual_size, None, None, false)?; let mut p_reader = PutObjReader::new(hash_reader); #[cfg(test)] fail_restore_multipart_at(RestoreMultipartFailurePoint::PutPart)?; + // `ropts` carries the object's ETag so the single-part copy-back + // keeps it (the writer only ever sees stored bytes). A part write + // must not inherit that object-level value, or every restored part + // would be recorded under the same ETag; each part keeps its own. + let mut part_write_opts = ropts.clone(); + part_write_opts.preserve_etag = Some(part_info.etag.clone()).filter(|etag| !etag.is_empty()); let p_info = self_ .clone() - .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &ropts) + .put_object_part(bucket, object, &res.upload_id, part_info.number, &mut p_reader, &part_write_opts) .await?; #[cfg(test)] let p_info = if restore_multipart_failure_is(RestoreMultipartFailurePoint::SizeMismatch) { @@ -9815,7 +9829,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } else { p_info }; - if p_info.size as i64 != part_info.actual_size { + if p_info.size as i64 != stored_part_size { return Err(Error::other(ObjectApiError::InvalidObjectState(GenericError { bucket: bucket.to_string(), object: object.to_string(), @@ -9847,6 +9861,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { user_defined: restore_commit_metadata, no_lock: false, decommission_capacity_admission: opts.decommission_capacity_admission.clone(), + // The composite ETag would otherwise be recomputed from the + // parts as they were written back, which for an encrypted or + // compressed object digests stored bytes rather than the + // object's public ETag. + preserve_etag: oi.etag.clone(), ..Default::default() }; self_ @@ -13488,10 +13507,317 @@ mod transition_commit_failure_tests { ); } + /// backlog#2368 B5: the tier stores opaque bytes, so the archive request + /// must not carry the object's encryption headers. Forwarding them made + /// every S3 target reject an SSE-C archive outright, asked the target to + /// encrypt an SSE-KMS object a second time under a key id it does not own, + /// and handed the wrapped DEK to a third-party provider. + #[tokio::test] + #[serial_test::serial] + async fn transition_does_not_forward_encryption_metadata_to_the_tier() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "transition-encryption-metadata-bucket"; + let object = "object.bin"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + let encryption_metadata = [ + ("x-amz-server-side-encryption", "aws:kms"), + ("x-amz-server-side-encryption-aws-kms-key-id", "arn:aws:kms:us-east-1:123:key/abc"), + ("x-amz-server-side-encryption-customer-algorithm", "AES256"), + (rustfs_utils::http::INTERNAL_ENCRYPTION_KEY_HEADER, "d3JhcHBlZC1kZWs="), + (rustfs_utils::http::INTERNAL_ENCRYPTION_IV_HEADER, "AAAAAAAAAAAAAAAA"), + (rustfs_utils::http::INTERNAL_ENCRYPTION_ALGORITHM_HEADER, "AES256"), + ]; + let mut user_defined: HashMap = encryption_metadata + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(); + user_defined.insert("x-amz-meta-owner".to_string(), "finance".to_string()); + + let mut reader = PutObjReader::from_vec(b"stored bytes the tier keeps opaque ".repeat(64)); + set_disks + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + user_defined, + ..Default::default() + }, + ) + .await + .expect("the encrypted source object should be written"); + let original = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("the source object should be readable"); + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object( + bucket, + object, + &ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("the encrypted object should transition"); + + let transitioned = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("the transitioned object should be readable"); + let remote_metadata = backend + .metadata(&transitioned.transitioned_object.name) + .await + .expect("the tier must have received the object"); + + for (key, _) in encryption_metadata { + assert!( + !remote_metadata.keys().any(|stored| stored.eq_ignore_ascii_case(key)), + "transition must not forward {key} to the tier: {remote_metadata:?}" + ); + } + assert!( + remote_metadata + .iter() + .any(|(key, value)| key.eq_ignore_ascii_case("x-amz-meta-owner") && value == "finance"), + "ordinary user metadata must still travel to the tier: {remote_metadata:?}" + ); + + // Read-through and restore both resolve encryption locally, so the + // stripped keys must survive untouched in the local metadata. + for (key, value) in encryption_metadata { + assert_eq!( + transitioned.user_defined.get(key).map(String::as_str), + Some(value), + "the local copy must keep {key}" + ); + } + } + + /// Deterministic bytes that do not repeat with a short period, so a slice + /// taken at the wrong offset cannot coincidentally compare equal. + fn stored_representation_bytes(seed: u32, len: usize) -> Vec { + (0..len) + .map(|index| { + let mixed = (index as u32).wrapping_add(seed).wrapping_mul(2_654_435_761); + (mixed >> 13) as u8 + }) + .collect() + } + + /// Compares two stored representations without dumping megabytes of bytes + /// into the failure output. + fn assert_stored_representation_eq(actual: &[u8], expected: &[u8], what: &str) { + assert_eq!(actual.len(), expected.len(), "{what}: stored length differs"); + if let Some(offset) = actual.iter().zip(expected).position(|(left, right)| left != right) { + panic!( + "{what}: stored bytes differ at offset {offset} (found {:#04x}, expected {:#04x})", + actual[offset], expected[offset] + ); + } + } + + async fn read_stored_representation(set_disks: &Arc, bucket: &str, object: &str) -> Vec { + let mut reader = set_disks + .get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await + .expect("stored-representation reader should open"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("stored body should drain"); + body + } + + /// backlog#2368 B3: the multipart restore loop addresses the tier in STORED + /// coordinates. Accumulating each part's PLAINTEXT length instead handed + /// every part a misaligned slice of the remote object whose length still + /// satisfied the range, the `HashReader` and the completion size check, so + /// the copy-back reported success while silently replacing the bytes. + /// + /// The fixture reproduces the encrypted geometry — a stored form LONGER + /// than the plaintext it encodes — because that is what keeps a + /// plaintext-coordinate range inside the tier object and makes the + /// corruption silent rather than a short read. + #[tokio::test] + #[serial_test::serial] + async fn multipart_restore_copies_the_stored_representation_back_verbatim() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "restore-multipart-stored-coordinates-bucket"; + let object = "object.bin"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + + // The minimum-part-size gate reads the PLAINTEXT length, so part one + // clears 5 MiB there while its stored form carries encoding overhead. + let part_shapes = [(6 * 1024 * 1024_usize, 9_216_usize), (256 * 1024_usize, 512_usize)]; + let mut user_defined = HashMap::new(); + user_defined.insert(rustfs_utils::http::INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "AES256".to_string()); + user_defined.insert( + rustfs_utils::http::INTERNAL_ENCRYPTION_IV_HEADER.to_string(), + "AAAAAAAAAAAAAAAA".to_string(), + ); + + let upload = set_disks + .new_multipart_upload( + bucket, + object, + &ObjectOptions { + user_defined: user_defined.clone(), + ..Default::default() + }, + ) + .await + .expect("multipart upload should be created"); + + let mut uploaded_parts = Vec::new(); + let mut expected_stored = Vec::new(); + for (index, (plaintext_len, overhead)) in part_shapes.iter().enumerate() { + let stored = stored_representation_bytes(index as u32 * 7 + 1, plaintext_len + overhead); + expected_stored.extend_from_slice(&stored); + let stored_len = stored.len() as i64; + let hash_reader = + HashReader::from_stream(std::io::Cursor::new(stored), stored_len, *plaintext_len as i64, None, None, false) + .expect("hash reader over the stored representation"); + let mut reader = PutObjReader::new(hash_reader); + let info = set_disks + .put_object_part(bucket, object, &upload.upload_id, index + 1, &mut reader, &ObjectOptions::default()) + .await + .expect("stored part should be staged"); + assert_eq!(info.size as i64, stored_len, "a part is stored in its encoded length"); + uploaded_parts.push(CompletePart { + part_num: info.part_num, + etag: info.etag, + ..Default::default() + }); + } + + let original = set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, uploaded_parts, &ObjectOptions::default()) + .await + .expect("source multipart upload should complete"); + let original_parts: Vec<(usize, usize, i64, String)> = original + .parts + .iter() + .map(|part| (part.number, part.size, part.actual_size, part.etag.clone())) + .collect(); + for (_, size, actual_size, _) in &original_parts { + assert!( + *size as i64 > *actual_size, + "the fixture must keep the two coordinate systems apart: stored {size} vs plaintext {actual_size}" + ); + } + let stored_before = read_stored_representation(&set_disks, bucket, object).await; + assert_stored_representation_eq(&stored_before, &expected_stored, "the fixture must store its encoded bytes verbatim"); + + let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); + register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; + set_disks + .transition_object( + bucket, + object, + &ObjectOptions { + no_lock: true, + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name, + etag: original.etag.clone().unwrap_or_default(), + ..Default::default() + }, + version_id: original.version_id.map(|version| version.to_string()), + mod_time: original.mod_time, + ..Default::default() + }, + ) + .await + .expect("multipart source should transition before restore"); + + let operation_id = Uuid::new_v4(); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(restore_metadata(operation_id, true)), + ..Default::default() + }, + ) + .await + .expect("the restore generation should be installed"); + let mut restore_opts = ObjectOptions::default(); + restore_opts.transition.restore_request.days = Some(1); + restore_opts.user_defined = restore_operation_id_metadata(operation_id); + set_disks + .clone() + .restore_transitioned_object(bucket, object, &restore_opts) + .await + .expect("multipart restore should complete"); + + let stored_after = read_stored_representation(&set_disks, bucket, object).await; + assert_stored_representation_eq( + &stored_after, + &expected_stored, + "a multipart restore must copy the stored representation back verbatim", + ); + + let restored = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("the restored object should be readable"); + let restored_parts: Vec<(usize, usize, i64, String)> = restored + .parts + .iter() + .map(|part| (part.number, part.size, part.actual_size, part.etag.clone())) + .collect(); + assert_eq!( + restored_parts, original_parts, + "restore must rebuild the same part layout, sizes and part ETags" + ); + assert_eq!(restored.size, original.size, "restore must keep the stored object size"); + // backlog#2369 P7.1: the copy-back digests stored bytes, so the object's + // public ETag has to be carried over rather than recomputed. + assert_eq!(restored.etag, original.etag, "restore must preserve the object ETag"); + } + #[tokio::test] #[serial_test::serial] async fn restore_failure_after_snapshot_cleans_exact_generation_and_returns_primary_error() { - let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + assert_restore_failure_cleanup_boundary(true).await; + } + + #[tokio::test] + #[serial_test::serial] + async fn restore_failure_after_snapshot_preserves_corrupt_known_transition_metadata() { + assert_restore_failure_cleanup_boundary(false).await; + } + + async fn assert_restore_failure_cleanup_boundary(legacy_unknown: bool) { + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; let bucket = "restore-post-snapshot-cleanup-bucket"; let object = "object.bin"; for disk in &disk_stores { @@ -13500,7 +13826,15 @@ mod transition_commit_failure_tests { let mut reader = PutObjReader::from_vec(b"post-snapshot cleanup source".repeat(1024)); let original = set_disks - .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + write_completion: WriteCompletion::TailDrained, + ..Default::default() + }, + ) .await .expect("source object should be written"); let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); @@ -13541,16 +13875,70 @@ mod transition_commit_failure_tests { .await .expect("transitioned metadata should be readable") .into_owned(); + let known_state = source_fi.transition_version_state; + assert_ne!(known_state, rustfs_filemeta::TransitionVersionState::Unknown); source_fi.metadata.extend(restore_metadata(operation_id, true)); - rustfs_utils::http::insert_str( - &mut source_fi.metadata, - rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, - "invalid".to_string(), - ); set_disks .update_object_meta(bucket, object, source_fi, &online_disks) .await - .expect("invalid backend identity fixture should be persisted"); + .expect("restore markers should be persisted"); + + // Normal writes reject damage to a reconciled binding. Model on-disk + // corruption directly, with and without the legacy missing-state field. + let mut corrupted_metadata = Vec::new(); + for temp_dir in &temp_dirs { + let metadata_path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE); + let encoded = tokio::fs::read(&metadata_path) + .await + .expect("transition metadata should be readable"); + let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode"); + let (version_index, mut version) = metadata + .find_version(original.version_id) + .expect("transitioned version should exist"); + let object_meta = version.object.as_mut().expect("transitioned version should be an object"); + rustfs_utils::http::insert_bytes( + &mut object_meta.meta_sys, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + b"invalid".to_vec(), + ); + if legacy_unknown { + rustfs_utils::http::remove_bytes( + &mut object_meta.meta_sys, + rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE, + ); + } + metadata.versions[version_index] = + rustfs_filemeta::FileMetaShallowVersion::try_from(version).expect("corrupt fixture should re-encode"); + tokio::fs::write(&metadata_path, metadata.marshal_msg().expect("corrupt fixture should encode")) + .await + .expect("corrupt fixture should be written"); + let persisted = tokio::fs::read(&metadata_path) + .await + .expect("corrupt fixture should be readable"); + let fixture = FileMeta::load(&persisted) + .expect("corrupt fixture should decode") + .find_version(original.version_id) + .expect("corrupt version should exist") + .1 + .into_fileinfo(bucket, object, true) + .expect("corrupt version should decode"); + assert_eq!( + fixture.transition_version_state, + if legacy_unknown { + rustfs_filemeta::TransitionVersionState::Unknown + } else { + known_state + } + ); + assert_eq!( + rustfs_utils::http::get_str(&fixture.metadata, rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID), + Some("invalid".to_string()) + ); + for (key, value) in restore_metadata(operation_id, true) { + assert_eq!(fixture.metadata.get(&key), Some(&value), "fixture must retain restore marker {key}"); + } + corrupted_metadata.push((metadata_path, persisted)); + } set_disks.invalidate_get_object_metadata_cache(bucket, object).await; let mut opts = ObjectOptions::default(); @@ -13573,6 +13961,23 @@ mod transition_commit_failure_tests { .await .expect("cleanup should leave the transitioned object readable"); assert_eq!(cleaned.transitioned_object.status, TRANSITION_COMPLETE); + if !legacy_unknown { + // Known bindings with corrupt identities must be repaired before + // cleanup; rejection must preserve both the binding and markers. + for (key, value) in restore_metadata(operation_id, true) { + assert_eq!(cleaned.user_defined.get(&key), Some(&value), "cleanup must preserve restore marker {key}"); + } + for (metadata_path, before) in corrupted_metadata { + assert_eq!( + tokio::fs::read(metadata_path) + .await + .expect("rejected cleanup metadata should remain readable"), + before, + "rejected cleanup must leave corrupt known metadata unchanged" + ); + } + return; + } assert!(!cleaned.user_defined.contains_key(s3s::header::X_AMZ_RESTORE.as_str())); assert!( rustfs_utils::http::get_str(cleaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,) diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 71c175b97..73aaa59af 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -770,12 +770,46 @@ impl ECStore { Ok(()) } + /// Prove a live bucket generation before repairing missing expansion volumes. + /// Unlike request validation, repair only needs one erasure set to confirm + /// existence; an incomplete expansion set is precisely what repair fixes. + /// Callers must hold the bucket namespace lock through the subsequent heal. + pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result { + let results = futures::future::join_all( + self.bucket_sets() + .map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }), + ) + .await; + let mut first_error = None; + for result in results { + match result { + Ok(_) => return Ok(true), + Err(err) if is_err_strict_volume_not_found(&err) => {} + Err(err) if first_error.is_none() => first_error = Some(err), + Err(_) => {} + } + } + match first_error { + Some(err) => Err(err), + None => Ok(false), + } + } + #[instrument(skip(self))] pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result { self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write) .await } + pub(crate) async fn get_bucket_info_from_sets_at_read_quorum( + &self, + bucket: &str, + opts: &BucketOptions, + ) -> Result { + self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read) + .await + } + async fn get_bucket_info_from_sets_with_quorum( &self, bucket: &str, @@ -2107,6 +2141,98 @@ mod tests { .expect("metadata initialization should recreate the bucket volume in the new pool"); } + #[tokio::test] + #[serial] + async fn bucket_metadata_init_repairs_half_created_expansion_pool() { + // Check both pool orders: an incomplete set must not hide a later + // complete set, and a complete set must not weaken request validation. + for complete_pool in 0..2 { + let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await; + let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple()); + for pool_index in 0..2 { + let present_disks = if pool_index == complete_pool { 4 } else { 2 }; + for disk_index in 0..present_disks { + tokio::fs::create_dir( + temp_dir + .path() + .join(format!("pool{pool_index}-disk{disk_index}")) + .join(&bucket), + ) + .await + .expect("fixture bucket volume should be created"); + } + } + assert_eq!( + ecstore + .get_bucket_info_from_sets(&bucket, &BucketOptions::default()) + .await + .expect_err("request validation must reject a half-created expansion set"), + StorageError::ErasureWriteQuorum + ); + + metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await; + + for pool_index in 0..2 { + for disk_index in 0..4 { + assert!( + temp_dir + .path() + .join(format!("pool{pool_index}-disk{disk_index}")) + .join(&bucket) + .is_dir(), + "metadata initialization must heal every missing expansion volume" + ); + } + } + ecstore + .get_bucket_info_from_sets(&bucket, &BucketOptions::default()) + .await + .expect("strict request validation should succeed after volume repair"); + } + } + + #[tokio::test] + #[serial] + async fn bucket_metadata_init_does_not_combine_partial_set_evidence() { + let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await; + let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple()); + for pool_index in 0..2 { + for disk_index in 0..2 { + tokio::fs::create_dir( + temp_dir + .path() + .join(format!("pool{pool_index}-disk{disk_index}")) + .join(&bucket), + ) + .await + .expect("fixture bucket volume should be created"); + } + } + assert_eq!( + ecstore + .bucket_exists_for_heal(&bucket) + .await + .expect_err("repair must require a complete quorum within one set"), + StorageError::ErasureWriteQuorum + ); + + metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await; + + for pool_index in 0..2 { + for disk_index in 0..4 { + assert_eq!( + temp_dir + .path() + .join(format!("pool{pool_index}-disk{disk_index}")) + .join(&bucket) + .is_dir(), + disk_index < 2, + "unproven bucket generations must not recreate missing volumes" + ); + } + } + } + #[tokio::test] #[serial] async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index d64083a39..c2e087acc 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -95,7 +95,6 @@ fn preflight_startup_rpc_secret_with( } } -const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3); const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30); const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30); const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5); @@ -280,20 +279,26 @@ where } } +async fn reconcile_local_decommission_after_init(store: &Arc, rx: CancellationToken) -> Result<()> { + store + .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery") + .await?; + if store.has_active_local_decommission_worker().await { + return Ok(()); + } + store.refresh_pool_status_meta().await?; + let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await); + if resume_required { + crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?; + } + store.spawn_missing_local_decommission_routines_with_token(rx).await +} + async fn supervise_local_decommission_after_init(store: Arc, rx: CancellationToken) { run_local_decommission_watchdog(rx.clone(), || { let store = store.clone(); let worker_rx = rx.clone(); - async move { - store - .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery") - .await?; - if store.has_active_local_decommission_worker().await { - return Ok(()); - } - store.refresh_pool_status_meta().await?; - store.spawn_missing_local_decommission_routines_with_token(worker_rx).await - } + async move { reconcile_local_decommission_after_init(&store, worker_rx).await } }) .await; } @@ -784,14 +789,9 @@ impl ECStore { ); } if has_local_decommission_leadership { - let store = self.clone(); - let decommission_rx = rx.clone(); - tokio::spawn(async move { - if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await { - return; - } - supervise_local_decommission_after_init(store, decommission_rx).await; - }); + // The watchdog checks recovery safety and retries transient failures. + // Resume persisted work without an unconditional cold-start delay. + tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone())); } let recovery_store = self.clone(); @@ -2104,6 +2104,44 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() { + let rx = CancellationToken::new(); + rx.cancel(); + run_local_decommission_watchdog(rx, || async { + panic!("cancelled startup must not schedule persisted work"); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; + mark_test_pool_decommissioning(&store, 0).await; + assert!(store.ctx.is_dist_erasure().await); + let worker_rx = CancellationToken::new(); + + { + let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test(); + let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone()) + .await + .expect_err("cold distributed recovery must wait for live fleet proof"); + assert!( + crate::core::pools::is_pool_activation_fleet_proof_error(&err), + "recovery must reach the live fleet proof gate: {err:?}" + ); + assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none)); + assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await)); + } + + super::reconcile_local_decommission_after_init(&store, worker_rx.clone()) + .await + .expect("restored fleet proof should admit the persisted worker"); + assert!(store.has_active_local_decommission_worker().await); + worker_rx.cancel(); + } + #[tokio::test(start_paused = true)] async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() { let rx = CancellationToken::new(); @@ -2115,11 +2153,13 @@ mod tests { let attempts = attempts.clone(); let rx = rx.clone(); async move { - if attempts.fetch_add(1, Ordering::SeqCst) == 0 { - Err(StorageError::SlowDown) - } else { - rx.cancel(); - Ok(()) + match attempts.fetch_add(1, Ordering::SeqCst) { + 0 => Err(StorageError::other("pool activation requires a live fleet capability proof")), + 1 => Err(StorageError::SlowDown), + _ => { + rx.cancel(); + Ok(()) + } } } } @@ -2128,8 +2168,11 @@ mod tests { tokio::task::yield_now().await; assert_eq!(attempts.load(Ordering::SeqCst), 1); tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await; - task.await.expect("watchdog task should exit after cancellation"); + tokio::task::yield_now().await; assert_eq!(attempts.load(Ordering::SeqCst), 2); + tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await; + task.await.expect("watchdog task should exit after cancellation"); + assert_eq!(attempts.load(Ordering::SeqCst), 3); } #[tokio::test(start_paused = true)] @@ -3664,6 +3707,31 @@ mod tests { OfflineTestDisks { disks } } + #[cfg(feature = "test-util")] + async fn force_set_disk_range_offline_for_test( + set: &Arc, + range: std::ops::Range, + ) -> OfflineTestDisks { + let disks = set + .disks + .read() + .await + .get(range) + .expect("offline test range must fit the set") + .iter() + .map(|disk| disk.clone().expect("fault-injection disk should start online")) + .collect::>(); + for disk in &disks { + disk.close().await.expect("fault injection should stop per-disk monitoring"); + disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline); + } + set.connect_disks().await; + for disk in &disks { + assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline); + } + OfflineTestDisks { disks } + } + fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta { let now = OffsetDateTime::now_utc(); let mut pool_stats = vec![RebalanceStats::default(); pool_count]; @@ -12760,6 +12828,422 @@ mod tests { body } + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() { + run_large_stack_async_test("legacy-state-reconcile-inspection", || { + legacy_transition_state_inspection_and_apply_case(false) + }); + } + + #[cfg(feature = "test-util")] + #[cfg(not(windows))] + #[test] + #[serial_test::serial(storage_class_env)] + fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() { + run_large_stack_async_test("legacy-state-reconcile-backfill", || { + legacy_transition_state_inspection_and_apply_case(true) + }); + } + + #[cfg(feature = "test-util")] + async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) { + #[cfg(windows)] + assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability"); + use crate::bucket::lifecycle::legacy_transition_state_reconcile::{ + LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest, + LegacyTransitionStateReconcileSelector, + }; + for (remote_version, expected_state) in [ + ("", rustfs_filemeta::TransitionVersionState::KnownDisabled), + ("null", rustfs_filemeta::TransitionVersionState::SuspendedNull), + ("opaque-version", rustfs_filemeta::TransitionVersionState::Exact), + ] { + let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + let tier_name = "LEGACY-RECONCILE"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + backend.set_put_remote_version(Some(remote_version.to_string())).await; + let bucket = "legacy-state-reconcile-bucket"; + let object = "archive.bin"; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("create legacy fixture bucket"); + let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024)); + let source = store + .put_object(bucket, object, &mut reader, &ObjectOptions::default()) + .await + .expect("write source"); + { + // Create the fixture under the existing remote-version writer + // gate. This does not authorize legacy metadata reconciliation. + let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test(); + temp_env::async_with_vars( + [ + (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")), + (rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")), + ], + store.transition_object( + bucket, + object, + &ObjectOptions { + transition: TransitionOptions { + status: TRANSITION_PENDING.to_string(), + tier: tier_name.to_string(), + etag: source.etag.clone().expect("source ETag"), + ..Default::default() + }, + mod_time: source.mod_time, + ..Default::default() + }, + ), + ) + .await + .expect("transition source"); + } + assert!( + crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof() + .await + .is_none(), + "fixture setup must not grant the missing reconciliation write capability" + ); + let selector = LegacyTransitionStateReconcileSelector { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: "null".to_string(), + }; + if expected_state == rustfs_filemeta::TransitionVersionState::Exact { + backend + .set_transition_candidate_probe_override(Some( + crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous, + )) + .await; + } + let converged = store + .inspect_legacy_transition_state(selector.clone()) + .await + .expect("inspect an already explicit transition"); + assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}"); + assert!(!converged.changed); + backend.set_transition_candidate_probe_override(None).await; + rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await; + let paths = (0..4) + .map(|disk| { + temp_dir + .path() + .join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}")) + }) + .collect::>(); + let mut original = Vec::new(); + for path in &paths { + original.push(tokio::fs::read(path).await.expect("original xl.meta")); + } + backend.clear_op_log().await; + let inspection = store.inspect_legacy_transition_state(selector.clone()); + assert!( + std::mem::size_of_val(&inspection) <= 4 * 1024, + "admin inspection future must remain stack-bounded" + ); + let inspected = inspection.await.expect("inspect legacy state"); + assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}"); + assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes"); + let target = inspected.target.expect("live probe should establish one model"); + assert_eq!(target.state, expected_state); + let request = LegacyTransitionStateReconcileRequest { + confirm: true, + selector, + source: inspected.source.expect("immutable source"), + original_sets: inspected.original_sets, + target, + reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"), + }; + #[cfg(not(windows))] + if write_enabled { + crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async { + crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None)); + let unsynced = store.reconcile_legacy_transition_state(request.clone()).await; + crate::disk::local::bucket_durability::set(bucket, None); + let unsynced = unsynced.expect("repair without metadata durability"); + assert_eq!(unsynced.outcome, Outcome::BackendUnavailable); + assert!(!unsynced.changed); + let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string()); + tokio::fs::create_dir(&rollback).await.expect("pending rollback directory"); + tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0]) + .await + .expect("pending old metadata backup"); + let unsettled = store.reconcile_legacy_transition_state(request.clone()).await; + tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback"); + let unsettled = unsettled.expect("repair must wait for rollback"); + assert_eq!(unsettled.outcome, Outcome::BackendUnavailable); + assert!(!unsettled.changed); + for (path, bytes) in paths.iter().zip(&original) { + assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes); + } + // The first disk commits; the second stops after staging. + // This models an interrupted cross-disk effect without rollback. + let disks = store.all_set_disks()[0].disk_inventory().await; + let first_disk = disks[0].as_ref().expect("first physical disk"); + let publication_path = first_disk + .get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}")) + .expect("local disk") + .expect("publication path"); + let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}"); + let _hook = crate::disk::os::prepared_publication_test_hooks::install_at( + crate::disk::os::prepared_publication_test_hooks::Stage::Rename, + &publication_path, + move || { + crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key); + }, + ); + let partial = store + .reconcile_legacy_transition_state(request.clone()) + .await + .expect("partial repair response"); + assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}"); + assert!(partial.changed, "first copy was committed: {partial:?}"); + assert!(partial.changes_indeterminate); + assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]); + for (path, bytes) in paths[1..].iter().zip(&original[1..]) { + assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes); + } + assert!( + store.all_set_disks()[0] + .load_file_info_versions_exact(bucket, object) + .await + .is_err(), + "cleanup cannot select a partial repair subset" + ); + let repaired = store + .reconcile_legacy_transition_state(request.clone()) + .await + .expect("retry original snapshot"); + assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}"); + assert!(repaired.changed); + assert!(!repaired.changes_indeterminate); + let mut committed = Vec::new(); + for (path, original) in paths.iter().zip(&original) { + let raw = tokio::fs::read(path).await.expect("repaired copy"); + let metadata = FileMeta::load(&raw).expect("decode repaired copy"); + let previous = FileMeta::load(original).expect("decode original copy"); + assert_eq!( + metadata.transition_reconcile_generation(None).unwrap(), + previous.transition_reconcile_generation(None).unwrap() + ); + let (_, version) = metadata.find_version(None).expect("selected version"); + let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo"); + assert_eq!(info.transition_version_state, expected_state); + assert_eq!(info.transition_version, request.target.remote_version); + committed.push(raw); + } + assert!( + store.all_set_disks()[0] + .load_file_info_versions_exact(bucket, object) + .await + .expect("converged cleanup snapshot") + .is_some() + ); + let replay = store + .reconcile_legacy_transition_state(request.clone()) + .await + .expect("idempotent original request replay"); + assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}"); + assert!(!replay.changed); + for (path, expected) in paths.iter().zip(&committed) { + assert_eq!( + tokio::fs::read(path).await.expect("replayed copy"), + *expected, + "idempotence preserves raw encoding" + ); + } + for (path, bytes) in paths.iter().zip(&original) { + tokio::fs::write(path, bytes) + .await + .expect("reset independent cancellation fixture"); + } + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, released) = std::sync::mpsc::channel::<()>(); + let _pause = crate::disk::os::prepared_publication_test_hooks::install_at( + crate::disk::os::prepared_publication_test_hooks::Stage::Rename, + &publication_path, + move || { + let _ = entered_tx.send(()); + let _ = released.recv(); + }, + ); + let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone())); + tokio::select! { + result = &mut repair => panic!("repair completed before publication pause: {result:?}"), + result = entered => result.expect("publication executor entered"), + } + let update_options = crate::disk::UpdateMetadataOpts::default(); + let mut update = Box::pin(first_disk.update_metadata( + bucket, + object, + FileInfo { + metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]), + ..Default::default() + }, + &update_options, + )); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut()) + .await + .is_err(), + "another metadata RMW must wait for publication" + ); + drop(repair); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut()) + .await + .is_err(), + "cancelling the coordinator must not release an in-flight disk mutation" + ); + release.send(()).expect("resume owned publication"); + update.await.expect("serialized metadata update"); + let raw = tokio::fs::read(&paths[0]) + .await + .expect("cancelled repair and later metadata update"); + let (_, version) = FileMeta::load(&raw) + .expect("metadata after cancellation") + .find_version(None) + .expect("selected version"); + let info = version + .into_fileinfo(bucket, object, true) + .expect("metadata after serialized update"); + assert_eq!(info.transition_version_state, expected_state); + assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept")); + let stale = store + .reconcile_legacy_transition_state(request.clone()) + .await + .expect("stale original request"); + assert_eq!( + stale.outcome, + Outcome::Corrupt, + "unrelated metadata change invalidates the original generation: {stale:?}" + ); + assert!(!stale.changed); + assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw); + assert_eq!(backend.remove_count().await, 0); + // A pinned version probe uses GET to verify that exact + // candidate; every backend operation still targets it. + let operations = backend.op_log().await; + assert!( + operations.iter().all(|operation| match operation { + MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object, + _ => false, + }), + "unexpected backend effects: {operations:?}" + ); + }) + .await; + continue; + } + let mut tampered = request.clone(); + tampered.source.remote_object.push_str("-other"); + let probes_before = backend.op_log().await.len(); + let rejected = store + .reconcile_legacy_transition_state(tampered) + .await + .expect("reject tampered tuple"); + assert_eq!(rejected.outcome, Outcome::Corrupt); + assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend"); + let applied = store + .reconcile_legacy_transition_state(request) + .await + .expect("apply must report unavailable write authority"); + assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}"); + assert_eq!(applied.reason_code, "write_fence_unavailable"); + assert!(!applied.changed); + for (path, expected) in paths.iter().zip(&original) { + assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected); + } + assert_eq!(backend.remove_count().await, 0); + assert!( + backend + .op_log() + .await + .iter() + .all(|operation| matches!(operation, MockWarmOp::Probe { .. })) + ); + + backend.set_unreachable(true).await; + let unavailable = store + .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: "null".to_string(), + }) + .await + .expect("unreachable tier is a diagnostic outcome"); + assert_eq!(unavailable.outcome, Outcome::BackendUnavailable); + assert!(!unavailable.changed); + backend.set_unreachable(false).await; + for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] { + backend + .set_transition_candidate_probe_override(Some( + crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()), + )) + .await; + let invalid_proof = store + .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: "null".to_string(), + }) + .await + .expect("invalid backend proof is a diagnostic outcome"); + assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}"); + assert!(invalid_proof.target.is_none()); + } + backend.set_transition_candidate_probe_override(None).await; + + backend.clear_op_log().await; + for path in &paths[1..] { + tokio::fs::remove_file(path) + .await + .expect("hide majority metadata copies in fixture"); + } + let minority = store + .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: "null".to_string(), + }) + .await + .expect("inspect minority legacy record"); + assert_eq!( + minority.outcome, + Outcome::BackendUnavailable, + "a minority owner must remain visible: {minority:?}" + ); + assert!( + backend.op_log().await.is_empty(), + "unproven metadata quorum cannot initiate a remote probe" + ); + for (path, bytes) in paths.iter().zip(&original) { + tokio::fs::write(path, bytes).await.expect("restore fixture copies"); + } + tokio::fs::write(&paths[0], b"corrupt-xl-meta") + .await + .expect("inject corrupt copy"); + let corrupt = store + .inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector { + bucket: bucket.to_string(), + object: object.to_string(), + version_id: "null".to_string(), + }) + .await + .expect("inspect corrupt legacy record"); + assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}"); + assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O"); + } + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] @@ -15202,25 +15686,24 @@ mod tests { assert!(deleted[0].found, "the aggregate error must retain the committed pool result"); drop(injection); - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let mut metadata_absent = true; - for pool in &store.pools { - metadata_absent &= pool - .get_disks_by_key(object) - .load_file_info_versions_exact(bucket, object) - .await - .expect("aggregate-error cleanup metadata should remain readable") - .is_none(); - } - if metadata_absent && backend.remove_count().await == 1 { - return; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("aggregate failure must not suppress committed receipt dispatch"); + // Exact reads can see subquorum metadata while workers remove each + // disk's free version. Inspect the final state after cleanup drains. + wait_for_expiry_workers_idle(&store).await; + for pool in &store.pools { + assert!( + pool.get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("aggregate-error cleanup metadata should remain readable") + .is_none(), + "aggregate failure must not suppress committed receipt cleanup" + ); + } + assert_eq!( + backend.remove_count().await, + 1, + "committed receipts must remove the shared remote object once" + ); assert_eq!(backend.object_count().await, 0, "the shared remote object should be removed exactly once"); store .delete_bucket(bucket, &DeleteBucketOptions::default()) @@ -16864,6 +17347,55 @@ mod tests { assert_eq!(body, original_body); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn object_lock_snapshot_uses_read_quorum_bucket_existence_probe() { + let temp = tempfile::tempdir().expect("create degraded snapshot store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp.path(), + "degraded-object-lock-snapshot", + &[(2, 12)], + CancellationToken::new(), + None, + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("degraded-ol-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create snapshot bucket"); + let expected_incarnation = store + .bucket_incarnation_id(&bucket) + .await + .expect("read bucket incarnation before degrading sets"); + + let mut offline_disks = Vec::new(); + for set in store.all_set_disks() { + offline_disks.push(force_set_disk_range_offline_for_test(&set, 6..12).await); + } + + let snapshot = store + .object_lock_config_snapshot(&bucket) + .await + .expect("read-quorum bucket existence should admit guarded Object Lock snapshot"); + assert!(matches!( + snapshot.state(), + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent + )); + assert!(snapshot.is_valid_for_destructive_put(store.id, &bucket, expected_incarnation)); + + let current_incarnation = crate::bucket::metadata_sys::get_object_lock_config_and_incarnation_from_disk_in(&ctx, &bucket) + .await + .expect("authoritative metadata read should also survive at read quorum") + .1; + assert_eq!(current_incarnation, expected_incarnation); + + drop(offline_disks); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn force_create_existing_bucket_preserves_incarnation_and_inflight_request() { @@ -18598,27 +19130,34 @@ mod tests { .await .expect("transition metadata should be readable"); let mut metadata = FileMeta::load(&encoded).expect("transition metadata should decode"); - let mut transitioned = metadata - .get_all_file_info_versions(bucket, object, true) - .expect("transitioned versions should decode") - .versions - .into_iter() - .find(|version| version.version_id == history.version_id) + let (version_index, mut transitioned) = metadata + .find_version(history.version_id) .expect("transitioned history should exist"); - transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown; - rustfs_utils::http::metadata_compat::remove_str( - &mut transitioned.metadata, + // Rewrite the serialized record to model legacy metadata; + // ordinary writes preserve an already reconciled state. + rustfs_utils::http::metadata_compat::remove_bytes( + &mut transitioned.object.as_mut().expect("history should be an object").meta_sys, rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE, ); - metadata - .add_version(transitioned) - .expect("unknown state should replace the transitioned version"); + metadata.versions[version_index] = rustfs_filemeta::FileMetaShallowVersion::try_from(transitioned) + .expect("legacy history should re-encode"); tokio::fs::write( &metadata_path, metadata.marshal_msg().expect("unknown transition metadata should encode"), ) .await .expect("unknown transition metadata should be written"); + let encoded = tokio::fs::read(&metadata_path) + .await + .expect("legacy transition metadata should be readable"); + let legacy = FileMeta::load(&encoded) + .expect("legacy transition metadata should decode") + .find_version(history.version_id) + .expect("legacy history should exist") + .1 + .into_fileinfo(bucket, object, true) + .expect("legacy history should decode"); + assert_eq!(legacy.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown); } let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event { action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction, diff --git a/crates/ecstore/src/store/list_objects.rs b/crates/ecstore/src/store/list_objects.rs index 82e6b1aa2..459ef0870 100644 --- a/crates/ecstore/src/store/list_objects.rs +++ b/crates/ecstore/src/store/list_objects.rs @@ -164,6 +164,16 @@ pub fn max_keys_plus_one(max_keys: i32, add_one: bool) -> i32 { max_keys } +fn list_versions_scan_limit(max_keys: i32, has_version_marker: bool) -> i32 { + if max_keys <= 0 { + return 0; + } + + // The marker object's versions may all be filtered out after gathering. + // Reserve its raw entry in addition to the next-page lookahead entry. + max_keys_plus_one(max_keys, true) + i32::from(has_version_marker) +} + #[derive(Debug, Clone, Copy, Eq, PartialEq)] enum GatherResultsState { LimitReached, @@ -2139,15 +2149,19 @@ fn build_list_versions_next_marker( // here; advertise it as the literal `null` marker so a resumed listing // parses it back to `VersionMarker::Null` instead of a nil UUID that // `find_version_index` can never match (issue #6745). - ( - Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)), + let version_marker = if last.is_dir && last.mod_time.is_none() { + // A CommonPrefix has no version to resume; a version marker would + // make the next page include this same prefix again. + None + } else { Some( last.version_id .filter(|v| !v.is_nil()) .map(|v| v.to_string()) .unwrap_or_else(|| "null".to_string()), - ), - ) + ) + }; + (Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)), version_marker) } else if let Some(last_prefix) = prefixes.last() { (Some(append_list_cache_id_to_marker(last_prefix.clone(), cache_id)), None) } else { @@ -2866,6 +2880,20 @@ fn listing_entries_supplement_target( return None; } + if let Some(directory) = entries.0.iter().flatten().find(|entry| entry.is_dir()) { + let directory_copies = entries + .0 + .iter() + .flatten() + .filter(|entry| entry.is_dir() && entry.name == directory.name) + .count(); + // A committed child may have some of its directory copies only on + // fallback disks, just like object metadata in a partial primary sample. + if directory_copies < resolver.dir_quorum { + return Some(directory.name.clone()); + } + } + for (idx, entry) in entries.0.iter().enumerate() { let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else { continue; @@ -4018,8 +4046,7 @@ impl ECStore { None }; - let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) }; - // Always request max_keys + 1 to detect if there are more results + let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker); let mut opts = ListPathOptions { bucket: bucket.to_owned(), prefix: prefix.to_owned(), @@ -5325,7 +5352,7 @@ impl Sets { None }; - let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) }; + let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker); let mut opts = ListPathOptions { bucket: bucket.to_owned(), prefix: prefix.to_owned(), @@ -6034,7 +6061,7 @@ impl SetDisks { let has_version_marker = version_marker.is_some(); let version_marker = version_marker.map(parse_version_marker).transpose()?; - let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) }; + let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker); let mut opts = ListPathOptions { bucket: bucket.to_owned(), prefix: prefix.to_owned(), @@ -6248,7 +6275,7 @@ impl SetDisks { None }; - let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) }; + let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker); let mut opts = ListPathOptions { bucket: bucket.to_owned(), prefix: prefix.to_owned(), @@ -7441,6 +7468,153 @@ mod test { assert!(cancel.is_cancelled()); } + #[test] + fn list_versions_pagination_scan_limit_boundaries() { + for has_version_marker in [false, true] { + assert_eq!(super::list_versions_scan_limit(-1, has_version_marker), 0); + assert_eq!(super::list_versions_scan_limit(0, has_version_marker), 0); + let marker_slot = i32::from(has_version_marker); + assert_eq!(super::list_versions_scan_limit(1, has_version_marker), 2 + marker_slot); + assert_eq!(super::list_versions_scan_limit(MAX_OBJECT_LIST, has_version_marker), 1001 + marker_slot); + assert_eq!(super::list_versions_scan_limit(i32::MAX, has_version_marker), 1001 + marker_slot); + } + } + + #[tokio::test] + async fn list_versions_pagination_does_not_require_an_empty_final_page() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; + + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "version-pagination-bucket"; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + store + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("pagination bucket should be created"); + let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + + for kind in ["objects", "deletes", "null", "mixed", "delimiter"] { + let count = if kind == "mixed" { 5 } else { 10 }; + let mut expected = Vec::new(); + for index in 0..count { + let name = if kind == "delimiter" && index % 2 == 1 { + format!("{kind}/testobject-{index:02}/child") + } else { + format!("{kind}/testobject-{index:02}") + }; + let entry = match kind { + "deletes" => test_delete_marker_meta_entry(&name, mod_time), + "null" => test_object_meta_entry(&name), + "mixed" => test_object_with_delete_marker_meta_entry(&name, mod_time, mod_time + time::Duration::SECOND), + _ => test_object_meta_entry_with_erasure_versions(&name, &[(mod_time, "etag", 2, 2)]), + }; + for dir in &dirs { + let object_dir = dir.path().join(bucket).join(&name); + tokio::fs::create_dir_all(&object_dir) + .await + .expect("pagination object directory should be created"); + tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), &entry.metadata) + .await + .expect("pagination metadata should be written"); + } + if kind == "delimiter" && index % 2 == 1 { + expected.push((name.trim_end_matches("child").to_owned(), None, false)); + } else { + let versions = entry.file_info_versions(bucket).expect("fixture versions should decode"); + expected.extend( + versions + .versions + .iter() + .map(|version| (name.clone(), version.version_id, version.deleted)), + ); + } + } + let prefix = format!("{kind}/"); + let delimiter = (kind == "delimiter").then(|| "/".to_owned()); + // Exercise each public/internal entry point with the reported page size. + // The store entry point also covers exact and one-over limit boundaries. + for (layer, max_keys) in [(0, 0), (0, 1), (0, 5), (0, 9), (0, 10), (0, 11), (1, 5), (2, 5), (3, 5)] { + if layer == 3 && delimiter.is_some() { + continue; + } + let mut marker = None; + let mut version_marker = None; + let expected_pages = if max_keys == 0 { + 1 + } else { + 10usize.div_ceil(usize::try_from(max_keys).expect("positive page size")) + }; + let mut actual = Vec::new(); + for page in 0..expected_pages { + let result = match layer { + 0 => { + store + .clone() + .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys) + .await + } + 1 => { + store.pools[0] + .clone() + .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys) + .await + } + 2 => { + store.pools[0].disk_set[0] + .clone() + .inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys) + .await + } + _ => { + store.pools[0].disk_set[0] + .clone() + .inner_list_object_versions_for_recursive_delete( + bucket, + &prefix, + marker, + version_marker, + max_keys, + ) + .await + } + } + .expect("version page should list successfully"); + let page_size = usize::try_from(max_keys).expect("nonnegative page size"); + assert_eq!(result.objects.len() + result.prefixes.len(), (10 - page * page_size).min(page_size)); + let has_more = page + 1 < expected_pages; + assert_eq!(result.is_truncated, has_more, "{kind}, layer {layer}, max_keys {max_keys}, page {page}"); + assert_eq!( + result.next_marker.is_some(), + has_more, + "key marker must exist only when another page exists" + ); + if !has_more { + assert!( + result.next_version_idmarker.is_none(), + "the final page must not advertise a version marker" + ); + } + actual.extend( + result + .objects + .into_iter() + .map(|object| (object.name, object.version_id, object.delete_marker)), + ); + actual.extend(result.prefixes.into_iter().map(|prefix| (prefix, None, false))); + marker = result.next_marker; + version_marker = result.next_version_idmarker; + } + // Objects and CommonPrefixes are serialized separately; compare their + // identities without relying on their relative position in the response. + actual.sort(); + let mut expected = if max_keys == 0 { Vec::new() } else { expected.clone() }; + expected.sort(); + assert_eq!(actual, expected, "{kind}, layer {layer}, max_keys {max_keys}"); + } + } + } + #[test] fn version_marker_is_applied_only_when_key_marker_entry_is_present() { let version_marker = Some(VersionMarker::Null); @@ -9448,6 +9622,71 @@ mod test { assert!(supplemented.is_latest_delete_marker()); } + #[tokio::test] + async fn latest_listing_supplement_checks_fallback_disks_for_common_prefix_quorum() { + let mut fallback_disks = Vec::new(); + let mut fallback_tempdirs = Vec::new(); + for index in 0..4 { + let tempdir = tempfile::tempdir().expect("fallback tempdir should be created"); + let endpoint = Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8")) + .expect("fallback endpoint should parse"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("fallback disk should be created"); + disk.make_volume("bucket").await.expect("fallback bucket should be created"); + for copies in [3, 4] { + if index < copies { + let object = format!("quux-{copies}/thud"); + let entry = test_object_meta_entry(&object); + disk.write_all("bucket", &format!("{object}/{STORAGE_FORMAT_FILE}"), bytes::Bytes::from(entry.metadata)) + .await + .expect("fallback child metadata should be written"); + } + } + fallback_disks.push(disk); + fallback_tempdirs.push(tempdir); + } + let supplement = ListingSupplement::new( + ListingSupplementOptions { + bucket: "bucket".to_owned(), + path: String::new(), + recursive: false, + incl_deleted: false, + skip_hidden_prefix_check: false, + filter_prefix: None, + forward_to: None, + per_disk_limit: 100, + skip_total_timeout: true, + walkdir_timeout: None, + walkdir_stall_timeout: None, + }, + Arc::new(fallback_disks), + FallbackClaimTracker::default(), + ); + // A 16-drive EC:4 set asks 12 primary disks. A committed write may + // exist on eight primary disks and all four remaining fallback disks. + let resolver = list_metadata_resolution_params("bucket".to_owned(), 4, 12, false, 0); + for fallback_copies in [3, 4] { + let prefix = format!("quux-{fallback_copies}/"); + let mut primary = vec![Some(test_dir_meta_entry(&prefix)); 8]; + primary.extend([None, None, None, None]); + let entry = + resolve_listing_entries_with_supplement(MetaCacheEntries(primary), resolver.clone(), true, supplement.clone()) + .await; + assert_eq!( + entry.map(|entry| entry.name), + (fallback_copies == 4).then_some(prefix), + "the common prefix needs all twelve copies, including fallback disks" + ); + } + } + #[test] fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() { let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 2f22066db..cec09eb60 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -50,7 +50,7 @@ use crate::services::notification_sys::{ use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata}; use crate::set_disk::{ SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, - is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain, + is_lock_optimization_enabled, is_object_lock_diag_enabled, }; use crate::storage_api_contracts::{ list::ListOperations as _, @@ -3079,12 +3079,7 @@ impl ECStore { let store = Arc::clone(self); let write = async move { let object = "buckets/.scanner-pause-backlog.json"; - let mut opts = ObjectOptions { - max_parity: true, - http_preconditions: Some(preconditions), - write_completion: crate::object_api::WriteCompletion::TailDrained, - ..Default::default() - }; + let mut opts = ObjectOptions::default(); // Match migration: fixed object namespace -> durable pool metadata -> // actual replica namespace. The replica need not be the hash-routed set. let object_guard = if store.single_pool() { @@ -3110,9 +3105,14 @@ impl ECStore { } else { None }; - let result = set - .put_object(RUSTFS_META_BUCKET, object, &mut PutObjReader::from_vec(data), &opts) - .await; + let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica( + set, + data, + preconditions, + opts, + "publish", + ) + .await; drop(capacity_guard); drop(object_guard); result @@ -3440,10 +3440,15 @@ impl ECStore { for pool in &self.pools { let hashed_set = pool.get_disks_by_key(object); - let lock_domain_already_held = !distributed - || locked_sets - .iter() - .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &hashed_set.lockers)); + let mut lock_domain_already_held = !distributed; + if !lock_domain_already_held { + for locked_set in &locked_sets { + if locked_set.shares_namespace_lock_domain(&hashed_set).await { + lock_domain_already_held = true; + break; + } + } + } if lock_domain_already_held { continue; } @@ -3500,10 +3505,15 @@ impl ECStore { let mut locked_sets = vec![fixed_set]; for pool in &self.pools { for set in &pool.disk_set { - let lock_domain_already_held = !distributed - || locked_sets - .iter() - .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers)); + let mut lock_domain_already_held = !distributed; + if !lock_domain_already_held { + for locked_set in &locked_sets { + if locked_set.shares_namespace_lock_domain(set).await { + lock_domain_already_held = true; + break; + } + } + } if lock_domain_already_held { continue; } @@ -3581,10 +3591,15 @@ impl ECStore { let mut locked_sets = vec![fixed_set]; for pool in &self.pools { for set in &pool.disk_set { - let lock_domain_already_held = !distributed - || locked_sets - .iter() - .any(|locked_set| same_distributed_lock_domain(&locked_set.lockers, &set.lockers)); + let mut lock_domain_already_held = !distributed; + if !lock_domain_already_held { + for locked_set in &locked_sets { + if locked_set.shares_namespace_lock_domain(set).await { + lock_domain_already_held = true; + break; + } + } + } if lock_domain_already_held { continue; } @@ -3667,11 +3682,15 @@ impl ECStore { .get(pool_idx) .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?; let set = pool.get_disks_by_key(object); - let lock_domain_already_held = !locked_sets.is_empty() - && (!distributed - || locked_sets.iter().any(|locked_set: &Arc| { - same_distributed_lock_domain(&locked_set.lockers, &set.lockers) - })); + let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed; + if !lock_domain_already_held { + for locked_set in &locked_sets { + if locked_set.shares_namespace_lock_domain(&set).await { + lock_domain_already_held = true; + break; + } + } + } if lock_domain_already_held { continue; } @@ -3747,29 +3766,37 @@ impl ECStore { opts: &ObjectOptions, no_lock: bool, ) -> Result { + let capacity_owner = DecommissionCapacityOwner::from_options(opts); match self .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock)) .await { - Ok((pinfo, _)) => Ok(pinfo.index), + Ok((pinfo, _)) => { + if let Some(owner) = capacity_owner { + if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? { + return Ok(pinfo.index); + } + } else { + return Ok(pinfo.index); + } + } Err(err) => { if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { return Err(err); } - - if let Some(owner) = DecommissionCapacityOwner::from_options(opts) { - let expected_data_bytes = opts - .capacity_expected_data_bytes() - .or_else(|| usize::try_from(size).ok()) - .unwrap_or_default(); - return self - .select_decommission_capacity_target_pool(owner, expected_data_bytes) - .await; - } - - self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull) } } + if let Some(owner) = capacity_owner { + let expected_data_bytes = opts + .capacity_expected_data_bytes() + .or_else(|| usize::try_from(size).ok()) + .unwrap_or_default(); + return self + .select_decommission_capacity_target_pool(owner, expected_data_bytes) + .await; + } + + self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull) } async fn find_data_movement_target_info( @@ -4251,8 +4278,10 @@ impl ECStore { } /// Return metadata for DELETE preflight, including an explicitly addressed - /// delete marker. Read APIs must keep using `get_object_info`; authorization - /// and Object Lock enforcement still belong to the caller and locked delete. + /// delete marker. GET/HEAD may also use this metadata-only lookup to enrich + /// an already failed read with marker headers, never to serve marker data. + /// Normal reads must keep using `get_object_info`; authorization and Object + /// Lock enforcement still belong to the caller and locked delete. #[instrument(level = "trace", skip_all)] pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_object_info_snapshot(bucket, object, opts, true).await @@ -5707,6 +5736,7 @@ mod tests { GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, }; + use crate::set_disk::same_distributed_lock_domain; use crate::set_disk::{SetDisks, disk_call_counters}; use crate::storage_api_contracts::bucket::MakeBucketOptions; use crate::storage_api_contracts::lifecycle::TransitionedObject; diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 18f5690d6..9549afcd5 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -50,6 +50,9 @@ use tracing::{error, warn}; use uuid::Uuid; use xxhash_rust::xxh64; +mod transition_reconcile; +pub use transition_reconcile::TransitionStateReconcileTarget; + // XL header specifies the format pub static XL_FILE_HEADER: [u8; 4] = *b"XL2 "; // pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4]; @@ -391,6 +394,16 @@ impl FileMeta { if ver_vid == fi_vid { let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?; + let previous = ver + .object + .as_ref() + .is_some_and(|object| { + rustfs_utils::http::contains_key_bytes( + &object.meta_sys, + rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID, + ) + }) + .then(|| ver.clone()); if let Some(ref mut obj) = ver.object { if replace_user_metadata { @@ -447,6 +460,9 @@ impl FileMeta { } } + if let Some(previous) = previous { + transition_reconcile::preserve_reconciled_transition(&previous, &mut ver)?; + } // Update version.header = ver.header(); version.meta = ver.marshal_msg()?; @@ -492,7 +508,7 @@ impl FileMeta { Ok(()) } - pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> { + pub fn add_version_filemata(&mut self, mut version: FileMetaVersion) -> Result<()> { if !version.valid() { return Err(Error::other("file meta version invalid")); } @@ -512,6 +528,7 @@ impl FileMeta { if existing.free_version() != version.free_version() { return Err(Error::other("cannot replace a free version with a non-free version")); } + transition_reconcile::preserve_reconciled_transition(&existing, &mut version)?; return self.set_idx(fidx, version); } diff --git a/crates/filemeta/src/filemeta/transition_reconcile.rs b/crates/filemeta/src/filemeta/transition_reconcile.rs new file mode 100644 index 000000000..10bd84d09 --- /dev/null +++ b/crates/filemeta/src/filemeta/transition_reconcile.rs @@ -0,0 +1,384 @@ +// Copyright 2026 RustFS Team +// SPDX-License-Identifier: Apache-2.0 + +use super::{FileMeta, FileMetaVersion}; +use crate::{Error, Result, TRANSITION_COMPLETE, TransitionVersionState}; +use rustfs_utils::http::metadata_compat::{ + SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes, + get_consistent_bytes, insert_bytes, remove_bytes, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +const RECONCILE_SUFFIXES: [&str; 3] = [ + SUFFIX_TRANSITIONED_VERSION_STATE, + SUFFIX_TRANSITIONED_VERSION_ID, + SUFFIX_TRANSITION_TIER_DESTINATION_ID, +]; + +/// The only fields a legacy transition repair is allowed to persist. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TransitionStateReconcileTarget { + pub state: TransitionVersionState, + pub remote_version: Option, + pub destination_id: String, +} + +impl TransitionStateReconcileTarget { + pub fn validate(&self) -> Result<()> { + let valid_version = match self.state { + TransitionVersionState::KnownDisabled => self.remote_version.is_none(), + TransitionVersionState::SuspendedNull => self.remote_version.as_deref() == Some("null"), + TransitionVersionState::Exact => self.remote_version.as_deref().is_some_and(|value| { + !value.is_empty() + && value.len() <= 1024 + && value != "null" + && !value.chars().any(char::is_control) + && !Uuid::parse_str(value).is_ok_and(|id| id.is_nil()) + }), + TransitionVersionState::Unknown => false, + }; + if !valid_version + || self.destination_id.len() != 64 + || !self + .destination_id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(Error::FileCorrupt); + } + Ok(()) + } +} + +impl FileMeta { + /// Canonical identity of every version and inline byte, excluding only the + /// three repairable suffixes on the selected version. It survives a repair + /// and encoding-order changes, while detecting unrelated metadata changes. + pub fn transition_reconcile_generation(&self, version_id: Option) -> Result> { + if self + .versions + .iter() + .filter(|version| version.header.version_id.unwrap_or_default() == version_id.unwrap_or_default()) + .count() + != 1 + { + return Err(Error::FileCorrupt); + } + let (selected, _) = self.find_version(version_id)?; + let mut versions = Vec::with_capacity(self.versions.len()); + for index in 0..self.versions.len() { + let mut version = self.get_idx(index)?; + if index == selected { + let object = version.object.as_mut().ok_or(Error::FileCorrupt)?; + for suffix in RECONCILE_SUFFIXES { + remove_bytes(&mut object.meta_sys, suffix); + } + } + versions.push(version); + } + versions.sort_by_key(|version| version.get_version_id().unwrap_or_default()); + let mut value = serde_json::to_value((&versions, &self.data)).map_err(|_| Error::FileCorrupt)?; + value.sort_all_objects(); + serde_json::to_vec(&value).map_err(|_| Error::FileCorrupt) + } + + /// Returns false for an already converged record. Callers must serialize + /// the read/check/commit and compare the observed metadata generation. + pub fn reconcile_transition_state( + &mut self, + version_id: Option, + target: &TransitionStateReconcileTarget, + ) -> Result { + target.validate()?; + let (index, mut version) = self.find_version(version_id)?; + let info = version.into_fileinfo("", "", true)?; + info.validate_for_metadata_read()?; + if info.transition_status != TRANSITION_COMPLETE + || info.transition_tier.is_empty() + || info.transitioned_objname.is_empty() + { + return Err(Error::FileCorrupt); + } + let object = version.object.as_mut().ok_or(Error::FileCorrupt)?; + let destination = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID); + if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) + && destination != Some(target.destination_id.as_bytes()) + { + return Err(Error::FileCorrupt); + } + if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) { + if info.transition_version_state == target.state + && info.transition_version == target.remote_version + && destination == Some(target.destination_id.as_bytes()) + { + return Ok(false); + } + return Err(Error::FileCorrupt); + } + if info.transition_version_state != TransitionVersionState::Unknown + || info + .transition_version + .as_deref() + .filter(|value| !value.is_empty()) + .is_some_and(|value| Some(value) != target.remote_version.as_deref()) + { + return Err(Error::FileCorrupt); + } + insert_bytes( + &mut object.meta_sys, + SUFFIX_TRANSITIONED_VERSION_STATE, + target.state.as_str().as_bytes().to_vec(), + ); + remove_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID); + if let Some(version) = &target.remote_version { + insert_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version.as_bytes().to_vec()); + } + insert_bytes( + &mut object.meta_sys, + SUFFIX_TRANSITION_TIER_DESTINATION_ID, + target.destination_id.as_bytes().to_vec(), + ); + version.into_fileinfo("", "", true)?.validate_for_metadata_read()?; + self.set_idx(index, version)?; + Ok(true) + } +} + +/// A stale healer or metadata writer may carry the original absent fields. +/// Preserve a proven binding for the same immutable transition, or reject an +/// attempted change of meaning. A new payload/version or a delete is separate. +pub(super) fn preserve_reconciled_transition(previous: &FileMetaVersion, next: &mut FileMetaVersion) -> Result<()> { + let (Some(previous_object), Some(next_object)) = (&previous.object, &mut next.object) else { + return Ok(()); + }; + if !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) + || !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) + || previous_object.data_dir != next_object.data_dir + { + return Ok(()); + } + let previous_info = previous.into_fileinfo("", "", true)?; + if previous_info.transition_version_state == TransitionVersionState::Unknown + || previous_info.transition_status != TRANSITION_COMPLETE + { + return Ok(()); + } + let next_info = next.into_fileinfo("", "", true)?; + if previous_info.transition_tier != next_info.transition_tier + || previous_info.transitioned_objname != next_info.transitioned_objname + || previous_info.transition_status != next_info.transition_status + || previous_info.size != next_info.size + || previous_info.metadata.get("etag") != next_info.metadata.get("etag") + { + return Err(Error::FileCorrupt); + } + let destination = + get_consistent_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID).ok_or(Error::FileCorrupt)?; + previous_info.validate_for_metadata_read()?; + TransitionStateReconcileTarget { + state: previous_info.transition_version_state, + remote_version: previous_info.transition_version.clone(), + destination_id: std::str::from_utf8(destination).map_err(|_| Error::FileCorrupt)?.to_string(), + } + .validate()?; + let next_object = next.object.as_mut().ok_or(Error::FileCorrupt)?; + if next_info + .transition_version + .as_ref() + .is_some_and(|version| Some(version) != previous_info.transition_version.as_ref()) + { + return Err(Error::FileCorrupt); + } + if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) { + if previous_info.transition_version_state != next_info.transition_version_state + || previous_info.transition_version != next_info.transition_version + { + return Err(Error::FileCorrupt); + } + } else if next_info.transition_version_state != TransitionVersionState::Unknown { + return Err(Error::FileCorrupt); + } + if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) + && get_consistent_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) != Some(destination) + { + return Err(Error::FileCorrupt); + } + for suffix in RECONCILE_SUFFIXES { + remove_bytes(&mut next_object.meta_sys, suffix); + if let Some(value) = get_consistent_bytes(&previous_object.meta_sys, suffix) { + insert_bytes(&mut next_object.meta_sys, suffix, value.to_vec()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ErasureInfo, FileInfo, ObjectPartInfo}; + + fn legacy() -> (FileMeta, FileInfo) { + let info = FileInfo { + version_id: Some(Uuid::from_u128(1)), + data_dir: Some(Uuid::from_u128(2)), + mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture time")), + size: 7, + parts: vec![ObjectPartInfo { + number: 1, + size: 7, + actual_size: 7, + ..Default::default() + }], + erasure: ErasureInfo { + algorithm: "ReedSolomon".to_string(), + data_blocks: 2, + parity_blocks: 2, + block_size: 1024 * 1024, + index: 1, + distribution: vec![1, 2, 3, 4], + ..Default::default() + }, + transition_status: TRANSITION_COMPLETE.to_string(), + transition_tier: "WARM".to_string(), + transitioned_objname: "remote-object".to_string(), + metadata: std::collections::HashMap::from([("etag".to_string(), "source-etag".to_string())]), + data: Some(bytes::Bytes::from_static(b"payload")), + ..Default::default() + }; + let mut metadata = FileMeta::new(); + metadata.add_version(info.clone()).expect("legacy fixture"); + (metadata, info) + } + + fn target(state: TransitionVersionState) -> TransitionStateReconcileTarget { + TransitionStateReconcileTarget { + state, + remote_version: match state { + TransitionVersionState::Exact => Some("opaque-version".to_string()), + TransitionVersionState::SuspendedNull => Some("null".to_string()), + _ => None, + }, + destination_id: "ab".repeat(32), + } + } + + #[test] + fn transition_reconcile_preserves_payload_and_generation_and_is_idempotent() { + for state in [ + TransitionVersionState::KnownDisabled, + TransitionVersionState::SuspendedNull, + TransitionVersionState::Exact, + ] { + let (mut metadata, info) = legacy(); + let mut other = info.clone(); + other.version_id = Some(Uuid::from_u128(3)); + other.data_dir = Some(Uuid::from_u128(4)); + other.transition_status.clear(); + other.transition_tier.clear(); + other.transitioned_objname.clear(); + other.data = Some(bytes::Bytes::from_static(b"other!!")); + metadata.add_version(other.clone()).expect("unrelated inline version"); + let other_before = metadata.find_version(other.version_id).expect("unrelated version").1; + let original_data = metadata.data.clone(); + let generation = metadata + .transition_reconcile_generation(info.version_id) + .expect("initial generation"); + let target = target(state); + assert!(metadata.reconcile_transition_state(info.version_id, &target).expect("repair")); + let bytes = metadata.marshal_msg().expect("encode repair"); + let mut reloaded = FileMeta::load(&bytes).expect("reload repair"); + assert_eq!(reloaded.data, original_data); + assert_eq!( + reloaded + .find_version(other.version_id) + .expect("preserved unrelated version") + .1, + other_before + ); + assert_eq!( + reloaded + .transition_reconcile_generation(info.version_id) + .expect("repaired generation"), + generation + ); + assert!(!reloaded.reconcile_transition_state(info.version_id, &target).expect("retry")); + let (_, version) = reloaded.find_version(info.version_id).expect("selected version"); + let repaired = version.into_fileinfo("", "", true).expect("decode explicit state"); + assert_eq!(repaired.transition_version_state, state); + assert_eq!(repaired.transition_version, target.remote_version); + assert_eq!(repaired.parts, info.parts); + for prefix in [ + rustfs_utils::http::RUSTFS_INTERNAL_PREFIX, + rustfs_utils::http::MINIO_INTERNAL_PREFIX, + ] { + assert_eq!( + version + .object + .as_ref() + .expect("object") + .meta_sys + .get(&format!("{prefix}{SUFFIX_TRANSITIONED_VERSION_STATE}")), + Some(&state.as_str().as_bytes().to_vec()) + ); + } + } + } + + #[test] + fn transition_reconcile_generation_detects_unrelated_metadata_and_inline_changes() { + let (mut metadata, info) = legacy(); + let original = metadata.transition_reconcile_generation(info.version_id).expect("generation"); + let mut updated = info.clone(); + updated.metadata.insert("user-tag".to_string(), "changed".to_string()); + metadata.update_object_version(updated).expect("update unrelated field"); + assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original); + let (mut metadata, mut info) = legacy(); + info.data = Some(bytes::Bytes::from_static(b"changed")); + metadata.add_version(info.clone()).expect("change inline bytes"); + assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original); + } + + #[test] + fn transition_reconcile_binding_survives_stale_heal_and_metadata_writes() { + let (mut metadata, mut stale) = legacy(); + let target = target(TransitionVersionState::Exact); + metadata + .reconcile_transition_state(stale.version_id, &target) + .expect("repair"); + metadata + .add_version(stale.clone()) + .expect("stale heal must preserve the binding"); + stale.metadata.insert("user-tag".to_string(), "updated".to_string()); + metadata + .update_object_version(stale.clone()) + .expect("ordinary metadata update"); + assert!( + !metadata + .reconcile_transition_state(stale.version_id, &target) + .expect("binding remains exact") + ); + } + + #[test] + fn transition_reconcile_rejects_explicit_unknown_and_conflicting_binding() { + let (mut metadata, mut info) = legacy(); + rustfs_utils::http::insert_str(&mut info.metadata, SUFFIX_TRANSITIONED_VERSION_STATE, "unknown".to_string()); + metadata.add_version(info.clone()).expect("explicit unknown fixture"); + assert!( + metadata + .reconcile_transition_state(info.version_id, &target(TransitionVersionState::Exact)) + .is_err() + ); + let (mut metadata, mut stale) = legacy(); + metadata + .reconcile_transition_state(stale.version_id, &target(TransitionVersionState::Exact)) + .expect("repair"); + stale.transition_version_state = TransitionVersionState::KnownDisabled; + assert!( + metadata.add_version(stale).is_err(), + "an explicit state cannot be replaced with a different model" + ); + } +} diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index 511595429..aabd39a6f 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -31,6 +31,7 @@ workspace = true [features] default = [] +test-util = [] hotpath = [ "hotpath/hotpath", "hotpath/tokio", @@ -104,6 +105,7 @@ walkdir = { workspace = true } http = { workspace = true } temp-env = { workspace = true, features = ["async_closure"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } +chrono = { workspace = true } [lib] doctest = false diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index d339a3b5a..b806990a2 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -847,7 +847,7 @@ mod tests { fn create_test_heal_manager() -> Arc { let storage: Arc = Arc::new(MockStorage); - Arc::new(HealManager::new(storage, None)) + Arc::new(HealManager::new_without_root_recovery_for_test(storage, None)) } #[test] @@ -1481,7 +1481,7 @@ mod tests { #[tokio::test] async fn test_process_start_request_returns_admission_result() { let storage: Arc = Arc::new(MockStorage); - let manager = Arc::new(HealManager::new( + let manager = Arc::new(HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -1826,7 +1826,7 @@ mod tests { #[tokio::test] async fn test_process_query_request_reports_displaced_terminal_detail() { - let heal_manager = Arc::new(HealManager::new( + let heal_manager = Arc::new(HealManager::new_without_root_recovery_for_test( Arc::new(MockStorage), Some(HealConfig { queue_size: 1, diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index e3d4e9b53..9cb5071bc 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -34,7 +34,7 @@ use std::time::{Duration, UNIX_EPOCH}; use tokio::sync::{RwLock, Semaphore}; use tracing::{debug, error, warn}; -use super::{DiskStore, EcstoreError}; +use super::{DiskStore, EcstoreError, POOL_META_NAME, RUSTFS_META_BUCKET}; /// Outcome of classifying an error returned by [`HealStorageAPI::heal_object`]. enum HealObjectOutcome { @@ -68,6 +68,13 @@ struct PageConcurrencyGuard { set_label: String, } +struct ErasureSetPassCounters<'a> { + processed_objects: &'a mut u64, + successful_objects: &'a mut u64, + failed_objects: &'a mut u64, + skipped_objects: &'a mut u64, +} + impl PageConcurrencyGuard { fn new(in_flight: Arc, set_label: String) -> Self { let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1; @@ -106,6 +113,7 @@ pub struct ErasureSetHealer { heal_opts: HealOpts, source: HealRequestSource, target_endpoints: Arc<[String]>, + pool_metadata_target_endpoints: Arc<[String]>, replacement_task_id: Option, replacement_target_identities: Option>, mainline_pacer: Option>, @@ -355,6 +363,7 @@ impl ErasureSetHealer { heal_opts, source, target_endpoints: Vec::new().into(), + pool_metadata_target_endpoints: Vec::new().into(), replacement_task_id: None, replacement_target_identities: None, mainline_pacer: None, @@ -378,6 +387,13 @@ impl ErasureSetHealer { self } + pub(crate) fn with_pool_metadata_targets(mut self, mut target_endpoints: Vec) -> Self { + target_endpoints.sort_unstable(); + target_endpoints.dedup(); + self.pool_metadata_target_endpoints = target_endpoints.into(); + self + } + pub(crate) fn with_replacement_identity_fence( mut self, replacement_target_identities: Option>, @@ -834,6 +850,21 @@ impl ErasureSetHealer { current_object_index = 0; } + if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 { + self.heal_replacement_pool_metadata( + set_disk_id, + &mut ErasureSetPassCounters { + processed_objects: &mut processed_objects, + successful_objects: &mut successful_objects, + failed_objects: &mut failed_objects, + skipped_objects: &mut skipped_objects, + }, + resume_manager, + checkpoint_manager, + ) + .await?; + } + // 5. finalize. Only declare the set healed when nothing failed AND // nothing was transiently skipped — otherwise the resume/checkpoint // state must survive so the failed/skipped versions are retried instead @@ -919,6 +950,207 @@ impl ErasureSetHealer { Ok(()) } + async fn heal_replacement_pool_metadata( + &self, + set_disk_id: &str, + counters: &mut ErasureSetPassCounters<'_>, + resume_manager: &ResumeManager, + checkpoint_manager: &CheckpointManager, + ) -> Result<()> { + let target_endpoints = if self.pool_metadata_target_endpoints.is_empty() { + self.target_endpoints.as_ref() + } else { + self.pool_metadata_target_endpoints.as_ref() + }; + let target_scoped_recreate = !self.heal_opts.dry_run && self.heal_opts.recreate && !target_endpoints.is_empty(); + if self.replacement_task_id.is_none() && !target_scoped_recreate { + return Ok(()); + } + if target_endpoints.is_empty() { + return Err(Error::TaskExecutionFailed { + message: "Replacement pool metadata heal requires target endpoints".to_string(), + }); + } + + let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}"); + let checkpoint_key = compose_key(&object_key, None); + let checkpoint = checkpoint_manager.get_checkpoint().await; + if checkpoint.processed_objects.contains(&checkpoint_key) + || checkpoint.failed_objects.contains(&checkpoint_key) + || checkpoint.skipped_objects.contains(&checkpoint_key) + { + return Ok(()); + } + drop(checkpoint); + + self.verify_replacement_identity_fence("pool metadata").await?; + resume_manager + .set_current_item(Some(RUSTFS_META_BUCKET.to_string()), Some(POOL_META_NAME.to_string())) + .await?; + + let result = match self + .storage + .heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, &self.heal_opts) + .await + { + Ok((result, None)) if target_outcomes_complete(&result, target_endpoints) => { + let object_size = result_object_size_u64(&result); + match self + .storage + .replacement_targets_have_version(RUSTFS_META_BUCKET, POOL_META_NAME, None, &self.heal_opts, target_endpoints) + .await + { + Ok(true) => (object_size, Ok(())), + Ok(false) => ( + object_size, + Err(Error::transient_skip( + "Skipped replacement pool metadata heal because target readback did not confirm the committed version", + )), + ), + Err(err) => ( + object_size, + Err(Error::transient_skip(format!( + "Skipped replacement pool metadata heal because target readback failed: {err}" + ))), + ), + } + } + Ok((result, None)) => ( + result_object_size_u64(&result), + Err(Error::transient_skip( + "Skipped replacement pool metadata heal because a replacement target was not committed", + )), + ), + Ok((result, Some(err))) => { + let object_size = result_object_size_u64(&result); + match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent | HealObjectOutcome::Transient => ( + object_size, + Err(Error::transient_skip(format!( + "Skipped replacement pool metadata heal due to transient error: {err}" + ))), + ), + HealObjectOutcome::Failed => (object_size, Err(err)), + } + } + Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), + Err(err) => match Self::classify_heal_object_error(&err) { + HealObjectOutcome::Absent | HealObjectOutcome::Transient => ( + 0, + Err(Error::transient_skip(format!( + "Skipped replacement pool metadata heal due to transient error: {err}" + ))), + ), + HealObjectOutcome::Failed => (0, Err(err)), + }, + }; + + let (object_size, result) = result; + let mut bytes_processed = self.progress.read().await.bytes_processed; + let mut telemetry_unknown = false; + let checkpoint_outcome = match result { + Ok(()) => { + telemetry_unknown |= !increment_counter(counters.successful_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); + debug!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket = RUSTFS_META_BUCKET, + object = POOL_META_NAME, + state = "healed", + "Replacement pool metadata healed" + ); + CheckpointObjectOutcome::Processed + } + Err(Error::TransientSkip { message }) => { + telemetry_unknown |= !increment_counter(counters.skipped_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); + warn!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket = RUSTFS_META_BUCKET, + object = POOL_META_NAME, + state = "transient_skip", + error = %message, + "Replacement pool metadata heal skipped due to transient error" + ); + CheckpointObjectOutcome::Skipped + } + Err(err) => { + telemetry_unknown |= !increment_counter(counters.failed_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); + warn!( + target: "rustfs::heal::erasure_healer", + event = EVENT_HEAL_ERASURE_OBJECT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_ERASURE_HEALER, + set_disk_id, + bucket = RUSTFS_META_BUCKET, + object = POOL_META_NAME, + state = "failed", + error = %err, + "Replacement pool metadata heal failed" + ); + CheckpointObjectOutcome::Failed + } + }; + + telemetry_unknown |= !increment_counter(counters.processed_objects); + let (outcome_record, counter_unknown, skipped_new_versions, skipped_ilm_expired) = { + let mut progress = self.progress.write().await; + progress.set_current_object(Some(object_key.clone())); + progress.update_object_progress( + *counters.processed_objects, + *counters.successful_objects, + *counters.failed_objects, + *counters.skipped_objects, + bytes_processed, + ); + if telemetry_unknown { + progress.mark_unknown(); + } + ( + CheckpointObjectOutcomeRecord { + object: checkpoint_key, + outcome: checkpoint_outcome, + successful: progress.objects_healed, + failed: progress.objects_failed, + skipped: progress.skipped_objects, + bytes: progress.bytes_processed, + skipped_new_versions: progress.skipped_new_versions, + skipped_ilm_expired: progress.skipped_ilm_expired, + counter_unknown: progress.counter_unknown, + }, + progress.counter_unknown, + progress.skipped_new_versions, + progress.skipped_ilm_expired, + ) + }; + checkpoint_manager.record_object_outcome(outcome_record).await?; + resume_manager + .update_progress_with_bytes( + *counters.processed_objects, + *counters.successful_objects, + *counters.failed_objects, + *counters.skipped_objects, + bytes_processed, + ) + .await?; + resume_manager + .set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired) + .await?; + if counter_unknown { + resume_manager.mark_counter_unknown().await?; + } + Ok(()) + } + /// heal single bucket with resume #[allow(clippy::too_many_arguments)] #[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))] @@ -1679,7 +1911,8 @@ mod resume_loop_tests { use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ - BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk, + BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, POOL_META_NAME, RUSTFS_META_BUCKET, + new_disk, }; use crate::{Error, Result}; use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource}; @@ -1773,6 +2006,28 @@ mod resume_loop_tests { assert!(!target_outcomes_complete(&duplicate, &["replacement-a".to_string()])); } + fn replacement_target_ok_result(endpoint: &str, object: &str) -> HealResultItem { + HealResultItem { + object: object.to_string(), + object_size: 1024, + before: Infos { + drives: vec![HealDriveInfo { + endpoint: endpoint.to_string(), + state: "missing".to_string(), + ..Default::default() + }], + }, + after: Infos { + drives: vec![HealDriveInfo { + endpoint: endpoint.to_string(), + state: "ok".to_string(), + ..Default::default() + }], + }, + ..Default::default() + } + } + #[derive(Clone)] struct Page { items: Vec, @@ -2444,6 +2699,8 @@ mod resume_loop_tests { HealRequestSource::AutoHeal, ) .with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone())); + env.storage + .set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME)); healer .heal_erasure_set(&["b".to_string()], "pool_0_set_0") @@ -2461,9 +2718,121 @@ mod resume_loop_tests { CheckpointManager::has_checkpoint(&env.healer.disk, &replacement_task_id).await, "the checkpoint must survive until the caller clears the healing marker" ); + assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]); drop(checkpoint); } + #[tokio::test] + async fn replacement_pool_metadata_readback_failure_schedules_retry() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let replacement_task_id = ResumeUtils::generate_task_id(); + ResumeManager::new_replacement_intent( + env.healer.disk.clone(), + replacement_task_id.clone(), + "pool_0_set_0".to_string(), + vec!["b".to_string()], + vec!["replacement-a".to_string()], + vec![crate::heal::resume::ReplacementTargetIdentity { + endpoint: "replacement-a".to_string(), + canonical_path: "/mnt/replacement-a".to_string(), + physical_device_ids: vec!["device-a".to_string()], + filesystem_identity: "1:2:3".to_string(), + }], + ) + .await + .expect("replacement intent should persist"); + env.storage + .set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME)); + env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false); + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts::default(), + HealRequestSource::AutoHeal, + ) + .with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone())); + + let error = healer + .heal_erasure_set(&["b".to_string()], "pool_0_set_0") + .await + .expect_err("unconfirmed pool metadata readback must keep the replacement incomplete"); + + assert!(error.to_string().contains("Replacement erasure set heal incomplete")); + let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id) + .await + .expect("replacement retry state must remain") + .get_state() + .await; + assert!(!state.completed); + assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent); + assert_eq!(state.retry_count, 1); + assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]); + } + + #[tokio::test] + async fn admin_recreate_target_heals_pool_metadata_before_completion() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts { + recreate: true, + pool: Some(0), + set: Some(0), + ..Default::default() + }, + HealRequestSource::Admin, + ) + .with_pool_metadata_targets(vec!["replacement-a".to_string()]); + env.storage + .set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME)); + + healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await + .expect("admin recreate should heal and verify pool metadata"); + + assert!(env.resume.get_state().await.completed); + assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]); + } + + #[tokio::test] + async fn admin_recreate_pool_metadata_readback_failure_keeps_resume_state() { + let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + let healer = ErasureSetHealer::new( + env.storage.clone(), + Arc::new(RwLock::new(HealProgress::new())), + CancellationToken::new(), + env.healer.disk.clone(), + HealOpts { + recreate: true, + pool: Some(0), + set: Some(0), + ..Default::default() + }, + HealRequestSource::Admin, + ) + .with_pool_metadata_targets(vec!["replacement-a".to_string()]); + env.storage + .set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME)); + env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false); + + let error = healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await + .expect_err("unconfirmed admin recreate pool metadata must keep the set incomplete"); + + assert!(error.to_string().contains("Erasure set heal incomplete")); + let state = env.resume.get_state().await; + assert!(!state.completed); + assert_eq!(state.retry_count, 1); + assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]); + } + #[tokio::test] async fn retry_exhaustion_keeps_resume_artifacts_for_recovery() { let env = make_env().await; diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index b54049a14..b269c446c 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -506,6 +506,25 @@ fn active_heal_for_dedup_key(active_heals: &HashMap>, key: .map(|(task_id, task)| (task_id.clone(), task.heal_type.clone())) } +fn request_matches_task(request: &HealRequest, task: &HealTask) -> bool { + request.heal_type == task.heal_type + && request.options == task.options + && request.priority == task.priority + && request.source == task.source + && request.retry_attempts == task.retry_attempts + && request.heal_endpoints == task.heal_endpoints +} + +fn request_matches_request(request: &HealRequest, existing: &HealRequest) -> bool { + request.heal_type == existing.heal_type + && request.options == existing.options + && request.priority == existing.priority + && request.source == existing.source + && request.force_start == existing.force_start + && request.retry_attempts == existing.retry_attempts + && request.heal_endpoints == existing.heal_endpoints +} + fn retrying_heal_for_dedup_key(retrying_heals: &HashMap, key: &str) -> Option<(String, HealType)> { retrying_heals .iter() @@ -517,14 +536,14 @@ fn completed_status_is_retrying(status: &HealTaskStatus) -> bool { matches!(status, HealTaskStatus::Retrying { .. }) } -fn retry_budget_for_result(task: &HealTask, result: &Result<()>) -> Option<(Duration, String)> { +fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch_failure: bool) -> Option<(Duration, String)> { let Err(err) = result else { return None; }; if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES { return None; } - if task.has_batch_failure() { + if task.has_batch_failure() && !retryable_batch_failure { return None; } @@ -540,12 +559,13 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>) -> Option<(Dura #[cfg(test)] fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> { - let (delay, error) = retry_budget_for_result(task, result)?; + let (delay, error) = retry_budget_for_result(task, result, false)?; Some((task.retry_request(), delay, error)) } async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> { - let (delay, error) = retry_budget_for_result(task, result)?; + let retryable_batch_failure = task.batch_failure_is_retryable().await; + let (delay, error) = retry_budget_for_result(task, result, retryable_batch_failure)?; let request = match task.retry_request_with_remaining_timeout().await { Ok(request) => request, Err(err) => { @@ -822,6 +842,10 @@ pub struct HealManager { replacement_recovery_anchors: Arc>>, /// Set IDs whose durable replacement metadata is corrupt or conflicting. replacement_recovery_blocked_sets: Arc>>, + /// Durable handoff of interrupted administrator root traversals. + root_recovery: Arc, + /// Keep forceStart's cancellation side effects inside the shutdown fence. + force_start_shutdown: Mutex<()>, /// Storage layer interface storage: Arc, /// Cancel token @@ -857,6 +881,7 @@ struct HealQueueContext<'a> { retrying_heals: &'a Arc>>, mrf_repair_notice_targets: &'a Arc>>>, replacement_recovery_anchors: &'a Arc>>, + root_recovery: &'a Arc, config: &'a Arc>, statistics: &'a Arc>, storage: &'a Arc, @@ -920,6 +945,10 @@ impl HealManager { matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal) } + fn queued_request_can_be_displaced(request: &HealRequest) -> bool { + !root_recovery::is_admin_heal_recovery(&request.heal_type, request.source) + } + fn request_bypasses_mainline_throttle(request: &HealRequest) -> bool { request.force_start || matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal) @@ -1056,11 +1085,15 @@ impl HealManager { let per_object_request = request.heal_type.is_per_object(); if queue_len >= queue_capacity && !request.force_start { - if Self::can_displace_queued_work(&request) && queue.can_displace_lower_priority(request.priority) { + if Self::can_displace_queued_work(&request) + && queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced) + { let request_id = request.id.clone(); let priority = request.priority; let source = request.source; - if let Some(displaced) = queue.push_displacing_lower_priority(request) { + if let Some(displaced) = + queue.push_displacing_lower_priority_where(request, Self::queued_request_can_be_displaced) + { publish_heal_queue_length(queue); Self::record_admission_metric(source, HealAdmissionResult::Accepted, context); demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", { @@ -1334,6 +1367,51 @@ impl HealManager { }); } + async fn start_root_recovery_terminal_gc(&self) { + let cancel = self.cancel_token.clone(); + let root_recovery = self.root_recovery.clone(); + tokio::spawn(async move { + let mut ticker = interval(RESUME_GC_INTERVAL); + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = ticker.tick() => { + match root_recovery.gc_terminal_receipts_once(SystemTime::now()).await { + Ok(report) => { + if report.pending_removed > 0 || report.terminals_removed > 0 || report.budget_exhausted { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_RESUME_GC, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + state = "root_terminal_gc", + scanned = report.scanned, + retained = report.retained, + pending_removed = report.pending_removed, + terminals_removed = report.terminals_removed, + budget_exhausted = report.budget_exhausted, + "Root heal terminal receipt GC inspected durable state" + ); + } + } + Err(error) => { + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_RESUME_GC, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + state = "root_terminal_gc_failed", + error = %error, + "Root heal terminal receipt GC failed" + ); + } + } + } + } + } + }); + } + /// Create new HealManager pub fn new(storage: Arc, config: Option) -> Self { Self::new_with_workload_provider(storage, config, None) @@ -1344,6 +1422,15 @@ impl HealManager { storage: Arc, config: Option, workload_provider: Option, + ) -> Self { + Self::new_with_root_recovery(storage, config, workload_provider, Arc::new(root_recovery::RootHealRecovery::default())) + } + + fn new_with_root_recovery( + storage: Arc, + config: Option, + workload_provider: Option, + root_recovery: Arc, ) -> Self { let config = config.unwrap_or_default(); Self { @@ -1358,6 +1445,8 @@ impl HealManager { mrf_repair_notice_targets: Arc::new(StdMutex::new(HashMap::new())), replacement_recovery_anchors: Arc::new(std::sync::Mutex::new(HashMap::new())), replacement_recovery_blocked_sets: Arc::new(std::sync::Mutex::new(HashSet::new())), + root_recovery, + force_start_shutdown: Mutex::new(()), storage, cancel_token: CancellationToken::new(), statistics: Arc::new(RwLock::new(HealStatistics::new())), @@ -1367,6 +1456,27 @@ impl HealManager { } } + #[cfg(any(test, feature = "test-util"))] + #[doc(hidden)] + pub fn new_without_root_recovery_for_test(storage: Arc, config: Option) -> Self { + Self::new_with_root_recovery(storage, config, None, Arc::new(root_recovery::RootHealRecovery::disabled_for_tests())) + } + + #[cfg(any(test, feature = "test-util"))] + #[doc(hidden)] + pub fn new_with_workload_provider_without_root_recovery_for_test( + storage: Arc, + config: Option, + workload_provider: Option, + ) -> Self { + Self::new_with_root_recovery( + storage, + config, + workload_provider, + Arc::new(root_recovery::RootHealRecovery::disabled_for_tests()), + ) + } + /// Start HealManager pub async fn start(&self) -> Result<()> { let mut state = self.state.write().await; @@ -1393,6 +1503,23 @@ impl HealManager { "Heal manager starting" ); + // Restore graceful-shutdown root responsibilities before automatic + // repair can admit overlapping work. + if let Err(error) = self.replay_root_heals().await { + // A missing owner or invalid root record must not block existing + // replacement recovery. Keep its file for a later restart after + // the owner is readable or the record has been repaired. + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_MANAGER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + state = "root_recovery_deferred", + error = %error, + "Root heal restart recovery deferred" + ); + } + // start scheduler self.start_scheduler().await?; @@ -1402,6 +1529,7 @@ impl HealManager { // Inspect resume artifacts in a bounded, fail-closed background task. self.start_resume_gc().await; + self.start_root_recovery_terminal_gc().await; // start auto disk scanner to heal unformatted disks if self.config.read().await.enable_auto_heal { @@ -1430,6 +1558,7 @@ impl HealManager { /// Stop HealManager pub async fn stop(&self) -> Result<()> { + let _force_start_guard = self.force_start_shutdown.lock().await; info!( target: "rustfs::heal::manager", event = EVENT_HEAL_MANAGER_STATE, @@ -1439,11 +1568,39 @@ impl HealManager { "Heal manager stopping" ); - // cancel all tasks - self.cancel_token.cancel(); - - // wait for all tasks to complete + // Keep scheduler, cancellation, and retry ownership stable until every + // unfinished admin control-plane heal has a durable successor. A failed + // write must leave the manager running and the shutdown marker unclean. let mut active_heals = self.active_heals.lock().await; + let queue = self.heal_queue.lock().await; + let retrying = self.retrying_heals.lock().await; + for task in active_heals.values() { + if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) { + if task.get_status().await == HealTaskStatus::Completed { + self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?; + } else { + let mut request = match task.retry_request_with_remaining_timeout().await { + Ok(request) => request, + Err(Error::TaskTimeout) => { + let mut request = task.retry_request(); + request.options.timeout = Some(Duration::ZERO); + request + } + Err(error) => return Err(error), + }; + request.retry_attempts = task.retry_attempts; + self.root_recovery.persist(&request).await?; + } + } + } + for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) { + self.root_recovery.persist(request).await?; + } + self.cancel_token.cancel(); + drop(retrying); + drop(queue); + + // cancel active workers after the durable handoff for task in active_heals.values() { if let Err(e) = task.cancel().await { warn!( @@ -1508,7 +1665,7 @@ impl HealManager { request: HealRequest, preserve_alias: bool, ) -> Result { - self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, preserve_alias, None) + self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, preserve_alias, true, None) .await } @@ -1544,19 +1701,43 @@ impl HealManager { request: HealRequest, mrf_notice_target: MrfRepairNoticeTarget, ) -> Result { - self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, Some(mrf_notice_target)) + self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, true, Some(mrf_notice_target)) .await } + pub(crate) async fn durable_mrf_repair_anchor( + &self, + intent: &rustfs_common::mrf_channel::MrfIntent, + ) -> Option { + match self.storage.mrf_bucket_incarnation_id(intent.bucket.as_ref()).await { + Ok(Some(bucket_incarnation_id)) => { + rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(intent, bucket_incarnation_id) + } + Ok(None) | Err(_) => None, + } + } + async fn submit_heal_request_with_receipt_alias_and_mrf_notice( &self, request: HealRequest, preserve_alias: bool, + accept_same_request_id_replay: bool, mrf_notice_target: Option, ) -> Result { let admission_start = Instant::now(); let source = request.source; let force_start = request.force_start; + // A forceStart must not retire an old durable owner if shutdown will + // reject its replacement. Hold the same gate through final admission. + let _force_start_guard = if source == HealRequestSource::Admin && force_start { + let guard = self.force_start_shutdown.lock().await; + if self.cancel_token.is_cancelled() { + return Err(Error::Other("Heal manager is stopping".to_string())); + } + Some(guard) + } else { + None + }; // HS-06 forceStart semantics (admin only): MinIO stops the old task // first and then starts the new one. Cancel any active admin task // overlapping this request's path before entering admission, so the @@ -1564,7 +1745,9 @@ impl HealManager { if request.source == HealRequestSource::Admin && request.force_start { let overlapping: Vec = { let active_heals = self.active_heals.lock().await; - active_heals + let queue = self.heal_queue.lock().await; + let retrying = self.retrying_heals.lock().await; + let mut ids = active_heals .iter() .filter(|(task_id, task)| { task.source == HealRequestSource::Admin @@ -1572,7 +1755,19 @@ impl HealManager { && *task_id != &request.id }) .map(|(task_id, _)| task_id.clone()) - .collect() + .collect::>(); + ids.extend( + queue + .requests() + .chain(retrying.values().map(|retrying| &retrying.request)) + .filter(|pending| { + root_recovery::is_admin_heal_recovery(&pending.heal_type, pending.source) + && heal_types_overlap(&request.heal_type, &pending.heal_type) != OverlapVerdict::Disjoint + && pending.id != request.id + }) + .map(|pending| pending.id.clone()), + ); + ids }; for task_id in overlapping { match self.cancel_task(&task_id).await { @@ -1586,17 +1781,16 @@ impl HealManager { result = "force_start_cancelled_overlap", "Admin forceStart cancelled an overlapping heal task" ), - Err(err) => warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %request.id, - cancelled_task_id = %task_id, - error = %err, - result = "force_start_cancel_failed", - "Admin forceStart failed to cancel an overlapping heal task" - ), + Err(err) => return Err(err), + } + } + // A failed or timed-out replay may have only its durable owner + // left. Cancel only records that overlap this forced start. + for pending in self.root_recovery.pending().await? { + if pending.id != request.id + && heal_types_overlap(&request.heal_type, &pending.heal_type) != OverlapVerdict::Disjoint + { + self.cancel_task(&pending.id).await?; } } } @@ -1609,10 +1803,75 @@ impl HealManager { // active -> retrying transitions can slip between duplicate checks. let lock_phase_start = Instant::now(); let active_heals = self.active_heals.lock().await; + if self.cancel_token.is_cancelled() { + return Err(Error::Other("Heal manager is stopping".to_string())); + } #[cfg(test)] pause_duplicate_admission_after_active_lock(&request.id).await; let mut queue = self.heal_queue.lock().await; let retrying_heals = self.retrying_heals.lock().await; + + let request_id_admission = active_heals + .get(&request.id) + .map(|task| (request_matches_task(&request, task), "active")) + .or_else(|| { + queue + .requests() + .find(|queued| queued.id == request.id) + .map(|queued| (request_matches_request(&request, queued), "queued")) + }) + .or_else(|| { + retrying_heals + .get(&request.id) + .map(|retrying| (request_matches_request(&request, &retrying.request), "retrying")) + }); + if let Some((matches_existing, duplicate_state)) = request_id_admission { + let admission = if matches_existing { + if accept_same_request_id_replay { + HealAdmissionResult::Accepted + } else { + Self::duplicate_admission_for_request(&request, &config) + } + } else { + HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning) + }; + if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged) + && let Some(target) = mrf_notice_target + { + let mut targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets); + Self::insert_mrf_repair_notice_target(&mut targets, &request.id, target); + } + drop(retrying_heals); + drop(queue); + drop(active_heals); + let lock_phase = lock_phase_start.elapsed(); + Self::record_admission_metric(request.source, admission, "duplicate"); + self.record_admission_observation(HealAdmissionObservation { + source, + result: admission, + context: "duplicate", + force_start, + displaced: false, + start_duration: admission_start.elapsed(), + lock_phase, + }); + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %request.id, + duplicate_state, + result = admission.result_label(), + reason = admission.reason_label(), + "Heal queue admission reused an existing request id" + ); + return Ok(HealAdmissionReceipt { + result: admission, + task_id: request.id, + }); + } + let duplicate = (!request.force_start).then(|| { active_heal_for_dedup_key(&active_heals, &dedup_key) .map(|(task_id, _)| (task_id, "active")) @@ -1760,9 +2019,29 @@ impl HealManager { } } + let durable_handoff = root_recovery::is_admin_heal_recovery(&request.heal_type, request.source); + let durable_handoff_required = durable_handoff + && (request.force_start + || queue.len() < config.queue_size + || (Self::can_displace_queued_work(&request) + && queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced))); + // An admin receipt is a control-plane responsibility. Persist it before + // queue publication so a crash after admission can replay it. + if durable_handoff_required { + self.root_recovery.persist(&request).await?; + } + let mut task_id = request.id.clone(); + let request_id = request.id.clone(); + let request_heal_type = request.heal_type.clone(); + let request_source = request.source; let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit"); let admission = admission_decision.result; + if durable_handoff_required && !admission.is_admitted() { + self.root_recovery + .remove(&request_id, &request_heal_type, request_source) + .await?; + } if admission == HealAdmissionResult::Merged && let Some(queued_id) = queue.queued_request_id_for_dedup_key(&dedup_key) { @@ -1819,7 +2098,10 @@ impl HealManager { /// Submit heal request. pub async fn submit_heal_request(&self, request: HealRequest) -> Result { - Ok(self.submit_heal_request_with_receipt_and_alias(request, true).await?.result) + Ok(self + .submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, false, None) + .await? + .result) } /// Get task status @@ -1830,7 +2112,7 @@ impl HealManager { /// Retrying, never Pending — then the queue, and finally a terminal /// completed entry. `heal_path` additionally constrains the map matches /// the way the `*_for_path` variants always have. - async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup { + async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result { let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path)); { @@ -1839,7 +2121,7 @@ impl HealManager { .get(canonical_task_id) .filter(|task| matches_path(&task.heal_type)) { - return TaskStateLookup::Active(Arc::clone(task)); + return Ok(TaskStateLookup::Active(Arc::clone(task))); } } @@ -1849,7 +2131,7 @@ impl HealManager { .get(canonical_task_id) .filter(|retrying| matches_path(&retrying.request.heal_type)) { - return TaskStateLookup::Retrying(retrying.status()); + return Ok(TaskStateLookup::Retrying(retrying.status())); } } @@ -1862,7 +2144,7 @@ impl HealManager { prune_completed_heal_statuses(&mut completed_heals); if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) { if completed_status_is_retrying(&completed.status) { - return TaskStateLookup::Completed(Arc::clone(completed)); + return Ok(TaskStateLookup::Completed(Arc::clone(completed))); } terminal_completed = Some(Arc::clone(completed)); } @@ -1875,7 +2157,7 @@ impl HealManager { None => queue.contains_request_id(canonical_task_id), }; if queued { - return TaskStateLookup::Queued; + return Ok(TaskStateLookup::Queued); } } @@ -1888,15 +2170,55 @@ impl HealManager { .cloned(); } - match terminal_completed { + if terminal_completed.is_none() + && let Some(completed) = self.root_recovery.completed(canonical_task_id).await? + && matches_path(&completed.heal_type) + { + terminal_completed = Some(Arc::new(completed)); + } + + Ok(match terminal_completed { Some(completed) => TaskStateLookup::Completed(completed), None => TaskStateLookup::NotFound, - } + }) + } + + async fn publish_admin_terminal( + &self, + task_id: &str, + heal_type: &HealType, + source: HealRequestSource, + completed: &CompletedHealStatus, + ) -> Result { + self.root_recovery + .persist_terminal(task_id, heal_type, source, completed) + .await + } + + async fn publish_admin_cancelled_terminal( + &self, + task_id: &str, + heal_type: &HealType, + source: HealRequestSource, + ) -> Result { + let completed = CompletedHealStatus { + outcome: None, + progress: None, + retained_bytes: std::sync::OnceLock::new(), + heal_type: heal_type.clone(), + status: HealTaskStatus::Cancelled, + result_items_truncated: false, + completed_at: SystemTime::now(), + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + }; + self.publish_admin_terminal(task_id, heal_type, source, &completed).await } pub async fn get_task_status(&self, task_id: &str) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - match self.lookup_task_state(&canonical_task_id, None).await { + match self.lookup_task_state(&canonical_task_id, None).await? { TaskStateLookup::Active(task) => Ok(task.get_status().await), TaskStateLookup::Retrying(status) => Ok(status), TaskStateLookup::Completed(completed) => Ok(completed.status.clone()), @@ -1916,7 +2238,7 @@ impl HealManager { /// full-snapshot semantics. pub async fn get_task_report_since(&self, task_id: &str, since: Option) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - match self.lookup_task_state(&canonical_task_id, None).await { + match self.lookup_task_state(&canonical_task_id, None).await? { TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await), TaskStateLookup::Retrying(status) => Ok(empty_task_report(status)), TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)), @@ -1939,7 +2261,7 @@ impl HealManager { since: Option, ) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await { + match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? { TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await), TaskStateLookup::Retrying(status) => Ok(empty_task_report(status)), TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)), @@ -1962,7 +2284,7 @@ impl HealManager { /// recently completed task, a different token is invalid for that path. pub async fn get_task_status_for_path(&self, heal_path: &str, task_id: &str) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await { + match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? { TaskStateLookup::Active(task) => Ok(task.get_status().await), TaskStateLookup::Retrying(status) => Ok(status), TaskStateLookup::Completed(completed) => Ok(completed.status.clone()), @@ -2016,11 +2338,18 @@ impl HealManager { } drop(completed_heals); - let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals); - prune_completed_heal_statuses(&mut displaced_terminals); - displaced_terminals - .values() - .any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path)) + { + let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals); + prune_completed_heal_statuses(&mut displaced_terminals); + if displaced_terminals + .values() + .any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path)) + { + return true; + } + } + + self.root_recovery.completed_matches_path(heal_path).await.unwrap_or(false) } /// Get task progress @@ -2032,7 +2361,7 @@ impl HealManager { pub async fn get_task_progress(&self, task_id: &str) -> Result { let canonical_task_id = self.canonical_task_id(task_id).await; - let progress = match self.lookup_task_state(&canonical_task_id, None).await { + let progress = match self.lookup_task_state(&canonical_task_id, None).await? { TaskStateLookup::Active(task) => Some(task.get_progress().await), TaskStateLookup::Completed(completed) => completed.progress.clone(), _ => None, @@ -2048,8 +2377,11 @@ impl HealManager { { let mut active_heals = self.active_heals.lock().await; if let Some(task) = active_heals.get(&canonical_task_id) { - task.cancel().await?; let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await; + self.publish_admin_terminal(&canonical_task_id, &task.heal_type, task.source, &completed) + .await?; + self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?; + task.cancel().await?; publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await; active_heals.remove(&canonical_task_id); publish_active_heal_count(&active_heals); @@ -2071,6 +2403,13 @@ impl HealManager { { let mut retrying_heals = self.retrying_heals.lock().await; + if let Some(retrying) = retrying_heals.get(&canonical_task_id) { + self.publish_admin_cancelled_terminal(&canonical_task_id, &retrying.request.heal_type, retrying.request.source) + .await?; + self.root_recovery + .remove(&canonical_task_id, &retrying.request.heal_type, retrying.request.source) + .await?; + } if let Some(retrying) = retrying_heals.remove(&canonical_task_id) { retrying.cancel_token.cancel(); drop(retrying_heals); @@ -2091,6 +2430,13 @@ impl HealManager { } let mut queue = self.heal_queue.lock().await; + if let Some(request) = queue.requests().find(|request| request.id == canonical_task_id) { + self.publish_admin_cancelled_terminal(&canonical_task_id, &request.heal_type, request.source) + .await?; + self.root_recovery + .remove(&request.id, &request.heal_type, request.source) + .await?; + } if queue.remove_request_id(&canonical_task_id).is_some() { publish_heal_queue_length(&queue); info!( @@ -2108,6 +2454,10 @@ impl HealManager { return Ok(()); } + drop(queue); + if self.root_recovery.cancel_pending(&canonical_task_id).await? { + return Ok(()); + } Err(Error::TaskNotFound { task_id: task_id.to_string(), }) @@ -2126,8 +2476,11 @@ impl HealManager { for task_id in &task_ids { if let Some(task) = active_heals.get(task_id) { - task.cancel().await?; let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await; + self.publish_admin_terminal(task_id, &task.heal_type, task.source, &completed) + .await?; + self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?; + task.cancel().await?; publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await; } active_heals.remove(task_id); @@ -2154,6 +2507,13 @@ impl HealManager { .collect::>(); for task_id in &task_ids { + if let Some(retrying) = retrying_heals.get(task_id) { + self.publish_admin_cancelled_terminal(task_id, &retrying.request.heal_type, retrying.request.source) + .await?; + self.root_recovery + .remove(task_id, &retrying.request.heal_type, retrying.request.source) + .await?; + } if let Some(retrying) = retrying_heals.remove(task_id) { retrying.cancel_token.cancel(); cancelled += 1; @@ -2175,6 +2535,21 @@ impl HealManager { } } + let queued_matches = { + let queue = self.heal_queue.lock().await; + queue + .requests() + .filter(|request| heal_type_matches_path(&request.heal_type, heal_path)) + .cloned() + .collect::>() + }; + for request in &queued_matches { + self.publish_admin_cancelled_terminal(&request.id, &request.heal_type, request.source) + .await?; + self.root_recovery + .remove(&request.id, &request.heal_type, request.source) + .await?; + } let mut queue = self.heal_queue.lock().await; let queued_cancelled = queue.remove_matching(|request| heal_type_matches_path(&request.heal_type, heal_path)); if !queued_cancelled.is_empty() { @@ -2187,6 +2562,17 @@ impl HealManager { self.remove_mrf_repair_notice_targets_for_task(&request.id); } + for pending in self + .root_recovery + .pending() + .await? + .into_iter() + .filter(|pending| heal_type_matches_path(&pending.heal_type, heal_path)) + { + if self.root_recovery.cancel_pending(&pending.id).await? { + cancelled += 1; + } + } if cancelled == 0 { return Err(Error::TaskNotFound { task_id: heal_path.to_string(), @@ -2298,6 +2684,7 @@ impl std::fmt::Debug for HealManager { mod auto_scan; mod queue; +mod root_recovery; mod scheduler; mod unclean_shutdown; diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index a47cd43e2..3e2883089 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -288,16 +288,29 @@ impl PriorityHealQueue { QueuePushOutcome::Accepted } - pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool { - self.heap.iter().any(|item| item.priority < priority) + pub(super) fn can_displace_lower_priority_where(&self, priority: HealPriority, can_displace: F) -> bool + where + F: Fn(&HealRequest) -> bool, + { + self.heap + .iter() + .any(|item| item.priority < priority && can_displace(&item.request)) } - pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option { + #[cfg(test)] + pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool { + self.can_displace_lower_priority_where(priority, |_| true) + } + + pub(super) fn push_displacing_lower_priority_where(&mut self, request: HealRequest, can_displace: F) -> Option + where + F: Fn(&HealRequest) -> bool, + { let mut retained = BinaryHeap::new(); let mut displaced: Option = None; while let Some(item) = self.heap.pop() { - if item.priority < request.priority { + if item.priority < request.priority && can_displace(&item.request) { let should_displace = displaced .as_ref() .map(|current| { @@ -337,6 +350,11 @@ impl PriorityHealQueue { displaced } + #[cfg(test)] + pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option { + self.push_displacing_lower_priority_where(request, |_| true) + } + /// Get statistics about queue contents by priority pub(super) fn get_priority_stats(&self) -> HashMap { let mut stats = HashMap::new(); diff --git a/crates/heal/src/heal/manager/root_recovery.rs b/crates/heal/src/heal/manager/root_recovery.rs new file mode 100644 index 000000000..85d0e3e89 --- /dev/null +++ b/crates/heal/src/heal/manager/root_recovery.rs @@ -0,0 +1,938 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Graceful-shutdown handoff for administrator heals. This namespace is +//! separate from erasure-set checkpoints and replacement generations, which +//! cannot represent an admitted admin control-plane request. One coordinator +//! disk owns each record; never create a fallback copy after an uncertain write +//! or deletion. + +use super::*; +use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes}; +use crate::heal::{DiskStore, RUSTFS_META_BUCKET}; +use serde::{Deserialize, Serialize}; + +// The metadata bucket already exists and its parent is durable. Creating a +// nested journal directory here would also require syncing every ancestor. +const ROOT_RECOVERY_PREFIX: &str = "root-heal-"; +const ROOT_TERMINAL_PREFIX: &str = "terminal-root-heal-"; +const LEGACY_ROOT_RECOVERY_SCHEMA: u32 = 1; +const ROOT_RECOVERY_SCHEMA: u32 = 2; +const ROOT_TERMINAL_SCHEMA: u32 = 1; +const ROOT_TERMINAL_GC_SCAN_BUDGET: usize = 1024; +const ROOT_TERMINAL_GC_DELETE_BUDGET: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum RecoveryHealType { + Cluster, + Bucket { + bucket: String, + }, + Object { + bucket: String, + object: String, + version_id: Option, + }, + Prefix { + bucket: String, + prefix: String, + }, + ErasureSet { + buckets: Vec, + set_disk_id: String, + }, + Metadata { + bucket: String, + object: String, + }, + EcDecode { + bucket: String, + object: String, + version_id: Option, + }, +} + +impl RecoveryHealType { + fn validate(&self) -> Result<()> { + match self { + Self::Cluster => {} + Self::Bucket { bucket } => validate_recovery_component("bucket", bucket)?, + Self::Object { + bucket, + object, + version_id, + } + | Self::EcDecode { + bucket, + object, + version_id, + } => { + validate_recovery_component("bucket", bucket)?; + validate_recovery_component("object", object)?; + if let Some(version_id) = version_id { + validate_recovery_component("version id", version_id)?; + } + } + Self::Prefix { bucket, prefix } => { + validate_recovery_component("bucket", bucket)?; + validate_recovery_component("prefix", prefix)?; + } + Self::ErasureSet { buckets, set_disk_id } => { + validate_recovery_component("set disk id", set_disk_id)?; + if buckets.is_empty() { + return Err(Error::Other("Admin heal recovery erasure set must name buckets".to_string())); + } + for bucket in buckets { + validate_recovery_component("bucket", bucket)?; + } + } + Self::Metadata { bucket, object } => { + validate_recovery_component("bucket", bucket)?; + validate_recovery_component("object", object)?; + } + } + Ok(()) + } +} + +impl From<&HealType> for RecoveryHealType { + fn from(heal_type: &HealType) -> Self { + match heal_type { + HealType::Cluster => Self::Cluster, + HealType::Bucket { bucket } => Self::Bucket { bucket: bucket.clone() }, + HealType::Object { + bucket, + object, + version_id, + } => Self::Object { + bucket: bucket.clone(), + object: object.clone(), + version_id: version_id.clone(), + }, + HealType::Prefix { bucket, prefix } => Self::Prefix { + bucket: bucket.clone(), + prefix: prefix.clone(), + }, + HealType::ErasureSet { buckets, set_disk_id } => Self::ErasureSet { + buckets: buckets.clone(), + set_disk_id: set_disk_id.clone(), + }, + HealType::Metadata { bucket, object } => Self::Metadata { + bucket: bucket.clone(), + object: object.clone(), + }, + HealType::ECDecode { + bucket, + object, + version_id, + } => Self::EcDecode { + bucket: bucket.clone(), + object: object.clone(), + version_id: version_id.clone(), + }, + } + } +} + +impl From for HealType { + fn from(heal_type: RecoveryHealType) -> Self { + match heal_type { + RecoveryHealType::Cluster => Self::Cluster, + RecoveryHealType::Bucket { bucket } => Self::Bucket { bucket }, + RecoveryHealType::Object { + bucket, + object, + version_id, + } => Self::Object { + bucket, + object, + version_id, + }, + RecoveryHealType::Prefix { bucket, prefix } => Self::Prefix { bucket, prefix }, + RecoveryHealType::ErasureSet { buckets, set_disk_id } => Self::ErasureSet { buckets, set_disk_id }, + RecoveryHealType::Metadata { bucket, object } => Self::Metadata { bucket, object }, + RecoveryHealType::EcDecode { + bucket, + object, + version_id, + } => Self::ECDecode { + bucket, + object, + version_id, + }, + } + } +} + +fn validate_recovery_component(label: &str, value: &str) -> Result<()> { + if value.is_empty() || value.contains('\0') { + return Err(Error::Other(format!("Invalid admin heal recovery {label}"))); + } + Ok(()) +} + +fn default_recovery_heal_type() -> RecoveryHealType { + RecoveryHealType::Cluster +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RootHealIntent { + schema: u32, + task_id: String, + #[serde(default = "default_recovery_heal_type")] + heal_type: RecoveryHealType, + #[serde(deserialize_with = "decode_options")] + options: HealOptions, + priority: HealPriority, + retry_attempts: u32, + created_at: SystemTime, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RootHealTerminal { + schema: u32, + task_id: String, + heal_type: RecoveryHealType, + status: HealTaskStatus, + progress: Option, + completed_at: SystemTime, +} + +impl RootHealTerminal { + fn from_completed(task_id: &str, completed: &CompletedHealStatus) -> Self { + Self { + schema: ROOT_TERMINAL_SCHEMA, + task_id: task_id.to_owned(), + heal_type: RecoveryHealType::from(&completed.heal_type), + status: completed.status.clone(), + progress: completed.progress.clone(), + completed_at: completed.completed_at, + } + } + + fn cancelled(task_id: &str, heal_type: &HealType) -> Self { + Self { + schema: ROOT_TERMINAL_SCHEMA, + task_id: task_id.to_owned(), + heal_type: RecoveryHealType::from(heal_type), + status: HealTaskStatus::Cancelled, + progress: None, + completed_at: SystemTime::now(), + } + } + + fn into_completed(self) -> CompletedHealStatus { + CompletedHealStatus { + outcome: None, + progress: self.progress, + retained_bytes: std::sync::OnceLock::new(), + heal_type: self.heal_type.into(), + status: self.status, + result_items_truncated: false, + completed_at: self.completed_at, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + } + } + + fn retained_at(&self, now: SystemTime) -> bool { + now.duration_since(self.completed_at) + .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION) + .unwrap_or(true) + } +} + +impl RootHealIntent { + fn from_request(request: &HealRequest) -> Self { + Self { + schema: ROOT_RECOVERY_SCHEMA, + task_id: request.id.clone(), + heal_type: RecoveryHealType::from(&request.heal_type), + options: request.options.clone(), + priority: request.priority, + retry_attempts: request.retry_attempts, + created_at: request.created_at, + } + } + + fn into_request(self) -> HealRequest { + let mut request = HealRequest::new(self.heal_type.into(), self.options, self.priority); + request.id = self.task_id; + request.source = HealRequestSource::Admin; + request.retry_attempts = self.retry_attempts; + request.created_at = self.created_at; + request + } +} + +#[derive(Default)] +pub(super) struct RootHealRecovery { + mutation: Mutex<()>, + #[cfg(any(test, feature = "test-util"))] + disabled_for_tests: bool, + #[cfg(test)] + disks: Option>, +} + +pub(super) fn is_admin_heal_recovery(heal_type: &HealType, source: HealRequestSource) -> bool { + source == HealRequestSource::Admin + && matches!( + heal_type, + HealType::Cluster + | HealType::Bucket { .. } + | HealType::Object { .. } + | HealType::Prefix { .. } + | HealType::ErasureSet { .. } + | HealType::Metadata { .. } + | HealType::ECDecode { .. } + ) +} + +fn decode_options<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result { + let value = serde_json::Value::deserialize(deserializer)?; + let object = value + .as_object() + .ok_or_else(|| serde::de::Error::custom("root heal options must be an object"))?; + const FIELDS: &[&str] = &[ + "scan_mode", + "remove_corrupted", + "recreate_missing", + "update_parity", + "recursive", + "dry_run", + "no_lock", + "timeout", + "pool_index", + "set_index", + ]; + if object.keys().any(|key| !FIELDS.contains(&key.as_str())) { + return Err(serde::de::Error::custom("unknown root heal recovery option")); + } + let options: HealOptions = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + if options.no_lock { + return Err(serde::de::Error::custom("administrator root heal cannot skip namespace locking")); + } + Ok(options) +} + +fn intent_path(task_id: &str) -> Result { + let parsed = uuid::Uuid::parse_str(task_id).map_err(|_| Error::Other("Invalid root heal recovery task id".to_string()))?; + if parsed.to_string() != task_id { + return Err(Error::Other("Noncanonical root heal recovery task id".to_string())); + } + Ok(format!("{ROOT_RECOVERY_PREFIX}{task_id}.json")) +} + +fn terminal_path(task_id: &str) -> Result { + let parsed = uuid::Uuid::parse_str(task_id).map_err(|_| Error::Other("Invalid root heal terminal task id".to_string()))?; + if parsed.to_string() != task_id { + return Err(Error::Other("Noncanonical root heal terminal task id".to_string())); + } + Ok(format!("{ROOT_TERMINAL_PREFIX}{task_id}.json")) +} + +fn decode_intent(task_id: &str, bytes: &[u8]) -> Result { + let _ = intent_path(task_id)?; + let intent: RootHealIntent = serde_json::from_slice(bytes) + .map_err(|error| Error::Other(format!("Invalid root heal recovery record {task_id}: {error}")))?; + if intent.task_id != task_id { + return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}"))); + } + match intent.schema { + LEGACY_ROOT_RECOVERY_SCHEMA if intent.heal_type == RecoveryHealType::Cluster => {} + ROOT_RECOVERY_SCHEMA => {} + _ => return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}"))), + } + intent.heal_type.validate()?; + Ok(intent) +} + +fn decode_terminal(task_id: &str, bytes: &[u8]) -> Result { + let _ = terminal_path(task_id)?; + let terminal: RootHealTerminal = serde_json::from_slice(bytes) + .map_err(|error| Error::Other(format!("Invalid root heal terminal record {task_id}: {error}")))?; + if terminal.schema != ROOT_TERMINAL_SCHEMA || terminal.task_id != task_id { + return Err(Error::Other(format!("Unsupported or mismatched root heal terminal record {task_id}"))); + } + terminal.heal_type.validate()?; + if !matches!( + terminal.status, + HealTaskStatus::Completed | HealTaskStatus::Cancelled | HealTaskStatus::Failed { .. } + ) { + return Err(Error::Other(format!("Non-terminal root heal receipt {task_id}"))); + } + Ok(terminal) +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(super) struct RootTerminalGcReport { + pub(super) scanned: usize, + pub(super) retained: usize, + pub(super) pending_removed: usize, + pub(super) terminals_removed: usize, + pub(super) budget_exhausted: bool, +} + +impl RootHealRecovery { + #[cfg(test)] + pub(super) fn with_disks(disks: Vec) -> Self { + Self { + mutation: Mutex::new(()), + disabled_for_tests: false, + disks: Some(disks), + } + } + + #[cfg(any(test, feature = "test-util"))] + pub(super) fn disabled_for_tests() -> Self { + Self { + mutation: Mutex::new(()), + disabled_for_tests: true, + #[cfg(test)] + disks: None, + } + } + + async fn disks(&self) -> Result> { + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(Vec::new()); + } + #[cfg(test)] + if let Some(disks) = &self.disks { + return Ok(disks.clone()); + } + let map = local_disk_map_read().await; + if map.values().any(Option::is_none) { + return Err(Error::Other("Root heal recovery owner may be on an unavailable local disk".to_string())); + } + let mut disks = map.values().flatten().cloned().collect::>(); + disks.sort_by_key(|disk| EcstoreDiskAPI::endpoint(disk.as_ref()).to_string()); + Ok(disks) + } + + async fn find(disks: &[DiskStore], task_id: &str) -> Result> { + let path = intent_path(task_id)?; + let mut found = None; + for disk in disks { + // read_all reports FileNotFound even when the whole metadata + // volume is absent; that is an unknown owner, not empty state. + EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?; + match EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, &path).await { + Ok(bytes) => { + decode_intent(task_id, &bytes)?; + if found.is_some() { + return Err(Error::Other(format!("Multiple root heal recovery owners for {task_id}"))); + } + found = Some((disk.clone(), bytes)); + } + Err(DiskError::FileNotFound) => {} + Err(error) => return Err(Error::Disk(error)), + } + } + Ok(found) + } + + async fn find_terminal(disks: &[DiskStore], task_id: &str) -> Result> { + let path = terminal_path(task_id)?; + let mut found = None; + for disk in disks { + EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?; + match EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, &path).await { + Ok(bytes) => { + decode_terminal(task_id, &bytes)?; + if found.is_some() { + return Err(Error::Other(format!("Multiple root heal terminal owners for {task_id}"))); + } + found = Some((disk.clone(), bytes)); + } + Err(DiskError::FileNotFound) => {} + Err(error) => return Err(Error::Disk(error)), + } + } + Ok(found) + } + + async fn find_retained_terminal( + disks: &[DiskStore], + task_id: &str, + now: SystemTime, + ) -> Result> { + let Some((disk, bytes)) = Self::find_terminal(disks, task_id).await? else { + return Ok(None); + }; + if decode_terminal(task_id, &bytes)?.retained_at(now) { + return Ok(Some((disk, bytes))); + } + Ok(None) + } + + async fn persist_terminal_locked( + disks: &[DiskStore], + task_id: &str, + terminal: RootHealTerminal, + ) -> Result> { + let path = terminal_path(task_id)?; + if let Some((_, bytes)) = Self::find_terminal(disks, task_id).await? { + let current = decode_terminal(task_id, &bytes)?; + if current == terminal { + return Self::find(disks, task_id).await; + } + return Err(Error::Other(format!("Root heal terminal record changed for {task_id}"))); + } + let pending = Self::find(disks, task_id).await?; + let disk = pending + .as_ref() + .map(|(disk, _)| disk.clone()) + .or_else(|| disks.first().cloned()) + .ok_or_else(|| Error::Other("No local disk available for root heal terminal receipt".to_string()))?; + let bytes = serde_json::to_vec(&terminal) + .map_err(|error| Error::Other(format!("Serialize root heal terminal receipt: {error}")))?; + match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, &path, None, Some(bytes.into())).await? { + EcstoreConditionalFileUpdate::Updated => Ok(pending), + _ => Err(Error::Other(format!("Root heal terminal record changed for {task_id}"))), + } + } + + pub(super) async fn persist(&self, request: &HealRequest) -> Result<()> { + if !is_admin_heal_recovery(&request.heal_type, request.source) { + return Ok(()); + } + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(()); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + let existing = Self::find(&disks, &request.id).await?; + let (disk, expected) = match existing { + Some((disk, bytes)) => (disk, Some(bytes)), + None => { + let disk = disks + .first() + .cloned() + .ok_or_else(|| Error::Other("No local disk available for root heal shutdown recovery".to_string()))?; + (disk, None) + } + }; + if request.options.no_lock { + return Err(Error::Other("Administrator root heal cannot skip namespace locking".to_string())); + } + let bytes = serde_json::to_vec(&RootHealIntent::from_request(request)) + .map_err(|error| Error::Other(format!("Serialize root heal recovery record: {error}")))?; + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(&request.id)?, + expected, + Some(bytes.into()), + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => Ok(()), + _ => Err(Error::Other(format!("Root heal recovery record changed for {}", request.id))), + } + } + + pub(super) async fn remove(&self, task_id: &str, heal_type: &HealType, source: HealRequestSource) -> Result { + if !is_admin_heal_recovery(heal_type, source) { + return Ok(false); + } + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(false); + } + self.remove_pending_by_id(task_id).await + } + + async fn remove_pending_by_id(&self, task_id: &str) -> Result { + let _guard = self.mutation.lock().await; + let Some((disk, bytes)) = Self::find(&self.disks().await?, task_id).await? else { + return Ok(false); + }; + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(task_id)?, + Some(bytes), + None, + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => Ok(true), + _ => Err(Error::Other(format!("Root heal recovery record changed while retiring {task_id}"))), + } + } + + pub(super) async fn checkpoint_failed_execution(&self, task: &HealTask) -> Result<()> { + if !is_admin_heal_recovery(&task.heal_type, task.source) { + return Ok(()); + } + let remaining = match task.retry_request_with_remaining_timeout().await { + Ok(request) => request.options.timeout, + Err(Error::TaskTimeout) => Some(Duration::ZERO), + Err(error) => return Err(error), + }; + let _guard = self.mutation.lock().await; + let Some((disk, expected)) = Self::find(&self.disks().await?, &task.id).await? else { + // A first execution that failed has no restart handoff to update. + return Ok(()); + }; + let mut intent = decode_intent(&task.id, &expected)?; + if HealType::from(intent.heal_type.clone()) != task.heal_type { + return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id))); + } + let mut expected_options = intent.options.clone(); + expected_options.timeout = task.options.timeout; + if intent.created_at != task.created_at || intent.priority != task.priority || expected_options != task.options { + return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id))); + } + // A terminal timeout leaves no runtime owner for stop() to snapshot. + // Checkpoint its consumed budget before publishing terminal status; + // never refund time if an earlier checkpoint is already stricter. + intent.options.timeout = match (intent.options.timeout, remaining) { + (Some(previous), Some(remaining)) => Some(previous.min(remaining)), + (previous, remaining) => previous.or(remaining), + }; + intent.retry_attempts = intent.retry_attempts.max(task.retry_attempts); + let bytes = serde_json::to_vec(&intent) + .map_err(|error| Error::Other(format!("Serialize root heal recovery checkpoint: {error}")))?; + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(&task.id)?, + Some(expected), + Some(bytes.into()), + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => Ok(()), + _ => Err(Error::Other(format!("Root heal recovery record changed while checkpointing {}", task.id))), + } + } + + pub(super) async fn cancel_pending(&self, task_id: &str) -> Result { + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(false); + } + if intent_path(task_id).is_err() { + return Ok(false); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + if Self::find_terminal(&disks, task_id).await?.is_some() { + if let Some((disk, bytes)) = Self::find(&disks, task_id).await? { + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(task_id)?, + Some(bytes), + None, + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => {} + _ => return Err(Error::Other(format!("Root heal recovery record changed while cancelling {task_id}"))), + } + } + return Ok(true); + } + let Some((disk, bytes)) = Self::find(&disks, task_id).await? else { + return Ok(false); + }; + let pending = decode_intent(task_id, &bytes)?; + let heal_type = HealType::from(pending.heal_type); + let terminal = RootHealTerminal::cancelled(task_id, &heal_type); + let _ = Self::persist_terminal_locked(&disks, task_id, terminal).await?; + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(task_id)?, + Some(bytes), + None, + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => Ok(true), + _ => Err(Error::Other(format!("Root heal recovery record changed while cancelling {task_id}"))), + } + } + + pub(super) async fn persist_terminal( + &self, + task_id: &str, + heal_type: &HealType, + source: HealRequestSource, + completed: &CompletedHealStatus, + ) -> Result { + if !is_admin_heal_recovery(heal_type, source) || completed.heal_type != *heal_type { + return Ok(false); + } + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(false); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + let pending = + Self::persist_terminal_locked(&disks, task_id, RootHealTerminal::from_completed(task_id, completed)).await?; + if let Some((disk, bytes)) = pending { + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(task_id)?, + Some(bytes), + None, + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => {} + _ => { + return Err(Error::Other(format!( + "Root heal recovery record changed while publishing terminal {task_id}" + ))); + } + } + } + Ok(true) + } + + pub(super) async fn completed(&self, task_id: &str) -> Result> { + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(None); + } + if terminal_path(task_id).is_err() { + return Ok(None); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + let Some((_, bytes)) = Self::find_retained_terminal(&disks, task_id, SystemTime::now()).await? else { + return Ok(None); + }; + Ok(Some(decode_terminal(task_id, &bytes)?.into_completed())) + } + + pub(super) async fn completed_matches_path(&self, heal_path: &str) -> Result { + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(false); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + for disk in &disks { + EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?; + let entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", -1).await { + Ok(entries) => entries, + Err(DiskError::FileNotFound) => continue, + Err(error) => return Err(Error::Disk(error)), + }; + for entry in entries { + let Some(task_id) = entry + .strip_prefix(ROOT_TERMINAL_PREFIX) + .and_then(|entry| entry.strip_suffix(".json")) + else { + continue; + }; + let Some((_, bytes)) = Self::find_retained_terminal(&disks, task_id, SystemTime::now()).await? else { + continue; + }; + let heal_type = HealType::from(decode_terminal(task_id, &bytes)?.heal_type); + if heal_type_matches_path(&heal_type, heal_path) { + return Ok(true); + } + } + } + Ok(false) + } + + pub(super) async fn gc_terminal_receipts_once(&self, now: SystemTime) -> Result { + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(RootTerminalGcReport::default()); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + let mut report = RootTerminalGcReport::default(); + let mut ids = HashSet::new(); + for disk in &disks { + if report.scanned >= ROOT_TERMINAL_GC_SCAN_BUDGET { + report.budget_exhausted = true; + break; + } + EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?; + let remaining = ROOT_TERMINAL_GC_SCAN_BUDGET.saturating_sub(report.scanned); + let count = i32::try_from(remaining).unwrap_or(i32::MAX); + let mut entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", count).await { + Ok(entries) => entries, + Err(DiskError::FileNotFound) => continue, + Err(error) => return Err(Error::Disk(error)), + }; + entries.sort_unstable(); + for entry in entries { + if report.scanned >= ROOT_TERMINAL_GC_SCAN_BUDGET { + report.budget_exhausted = true; + break; + } + report.scanned += 1; + let Some(task_id) = entry + .strip_prefix(ROOT_TERMINAL_PREFIX) + .and_then(|entry| entry.strip_suffix(".json")) + else { + continue; + }; + let _ = terminal_path(task_id)?; + ids.insert(task_id.to_string()); + } + } + + let mut ids = ids.into_iter().collect::>(); + ids.sort(); + let mut deletes = 0usize; + for task_id in ids { + if deletes >= ROOT_TERMINAL_GC_DELETE_BUDGET { + report.budget_exhausted = true; + break; + } + let Some((terminal_disk, terminal_bytes)) = Self::find_terminal(&disks, &task_id).await? else { + continue; + }; + let terminal = decode_terminal(&task_id, &terminal_bytes)?; + if terminal.retained_at(now) { + report.retained += 1; + continue; + } + if let Some((pending_disk, pending_bytes)) = Self::find(&disks, &task_id).await? { + match EcstoreDiskAPI::compare_and_update_file( + pending_disk.as_ref(), + RUSTFS_META_BUCKET, + &intent_path(&task_id)?, + Some(pending_bytes), + None, + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => { + deletes += 1; + report.pending_removed += 1; + report.retained += 1; + continue; + } + _ => { + return Err(Error::Other(format!( + "Root heal recovery record changed while pruning terminal receipt {task_id}" + ))); + } + } + } + match EcstoreDiskAPI::compare_and_update_file( + terminal_disk.as_ref(), + RUSTFS_META_BUCKET, + &terminal_path(&task_id)?, + Some(terminal_bytes), + None, + ) + .await? + { + EcstoreConditionalFileUpdate::Updated => { + deletes += 1; + report.terminals_removed += 1; + } + _ => return Err(Error::Other(format!("Root heal terminal record changed while pruning {task_id}"))), + } + } + Ok(report) + } + + pub(super) async fn pending(&self) -> Result> { + #[cfg(any(test, feature = "test-util"))] + if self.disabled_for_tests { + return Ok(Vec::new()); + } + let _guard = self.mutation.lock().await; + let disks = self.disks().await?; + let mut ids = HashSet::new(); + for disk in &disks { + EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?; + let entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", -1).await { + Ok(entries) => entries, + Err(DiskError::FileNotFound) => continue, + Err(error) => return Err(Error::Disk(error)), + }; + for entry in entries { + let Some(task_id) = entry + .strip_prefix(ROOT_RECOVERY_PREFIX) + .and_then(|entry| entry.strip_suffix(".json")) + else { + continue; + }; + let _ = intent_path(task_id)?; + ids.insert(task_id.to_string()); + } + } + let mut requests = Vec::new(); + for task_id in ids { + if Self::find_terminal(&disks, &task_id).await?.is_some() { + continue; + } + if let Some((_, bytes)) = Self::find(&disks, &task_id).await? { + requests.push(decode_intent(&task_id, &bytes)?.into_request()); + } + } + requests.sort_by(|left, right| left.created_at.cmp(&right.created_at).then_with(|| left.id.cmp(&right.id))); + Ok(requests) + } +} + +impl HealManager { + pub(super) async fn replay_root_heals(&self) -> Result<()> { + // Decode every record before admitting anything. These are already + // accepted responsibilities, so restore distinct IDs even when their + // paths overlap or the configured admission capacity has changed. + let requests = self.root_recovery.pending().await?; + let active = self.active_heals.lock().await; + let mut queue = self.heal_queue.lock().await; + let retrying = self.retrying_heals.lock().await; + for mut request in requests { + request.force_start = true; + let existing = active + .get(&request.id) + .map(|task| request_matches_task(&request, task)) + .or_else(|| { + queue + .requests() + .find(|queued| queued.id == request.id) + .map(|queued| request_matches_request(&request, queued)) + }) + .or_else(|| { + retrying + .get(&request.id) + .map(|retrying| request_matches_request(&request, &retrying.request)) + }); + match existing { + Some(true) => continue, + Some(false) => return Err(Error::Other(format!("Conflicting root heal recovery task {}", request.id))), + None => {} + } + queue.push(request); + } + publish_heal_queue_length(&queue); + Ok(()) + } +} diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index c9ea5a8a7..dfe96dfaa 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -26,6 +26,7 @@ impl HealManager { let retrying_heals = self.retrying_heals.clone(); let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone(); let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); + let root_recovery = self.root_recovery.clone(); let cancel_token = self.cancel_token.clone(); let statistics = self.statistics.clone(); let storage = self.storage.clone(); @@ -59,6 +60,7 @@ impl HealManager { retrying_heals: &retrying_heals, mrf_repair_notice_targets: &mrf_repair_notice_targets, replacement_recovery_anchors: &replacement_recovery_anchors, + root_recovery: &root_recovery, config: &config, statistics: &statistics, storage: &storage, @@ -78,6 +80,7 @@ impl HealManager { retrying_heals: &retrying_heals, mrf_repair_notice_targets: &mrf_repair_notice_targets, replacement_recovery_anchors: &replacement_recovery_anchors, + root_recovery: &root_recovery, config: &config, statistics: &statistics, storage: &storage, @@ -106,6 +109,7 @@ impl HealManager { retrying_heals, mrf_repair_notice_targets, replacement_recovery_anchors, + root_recovery, config, statistics, storage, @@ -117,6 +121,9 @@ impl HealManager { let config = config.read().await; let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider); let mut active_heals_guard = active_heals.lock().await; + if cancel_token.is_cancelled() { + return; + } publish_active_heal_count(&active_heals_guard); // Check if new heal tasks can be started @@ -206,6 +213,7 @@ impl HealManager { let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone(); let statistics_clone = statistics.clone(); let notify_clone = notify.clone(); + let root_recovery_clone = root_recovery.clone(); let manager_cancel_token = cancel_token.clone(); let task_type_label_for_spawn = task_type_label.clone(); let task_set_label_for_spawn = task_set_label.clone(); @@ -312,7 +320,47 @@ impl HealManager { completed_status_entry.status = HealTaskStatus::Cancelled; completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await)); } - let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); + let terminal_completion = matches!( + completed_status, + HealTaskStatus::Completed | HealTaskStatus::Cancelled | HealTaskStatus::Failed { .. } + ); + if owns_completion + && terminal_completion + && root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) + && let Err(error) = root_recovery_clone + .persist_terminal(&task_id, &task.heal_type, task.source, &completed_status_entry) + .await + { + // Keep the durable responsibility if terminal + // publication fails. Replaying the task is preferable + // to losing the final receipt across restart. + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + task_id, + state = "root_recovery_terminal_publish_failed", + error = %error, + "Failed to publish heal terminal receipt" + ); + } + if owns_completion + && !terminal_completion + && result.is_err() + && let Err(error) = root_recovery_clone.checkpoint_failed_execution(&task).await + { + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + task_id, + state = "root_recovery_checkpoint_failed", + error = %error, + "Failed to checkpoint root heal recovery execution budget" + ); + } let completed_status_for_verified_events = completed_status_entry.clone(); // Keep retry ownership continuous: status snapshots acquire // these locks in the same active -> retrying order. @@ -733,23 +781,30 @@ pub(super) fn mrf_verified_repair_event_for_target( HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent, _ => return None, }; - if target.kind != MrfKind::PartialWrite { - return None; - } - let expected_kind = HealObjectKind::Object; + let expected_kind = match target.kind { + MrfKind::DecodeFailure => HealObjectKind::Decode, + MrfKind::MetadataCorruption => HealObjectKind::Metadata, + MrfKind::PartialWrite => HealObjectKind::Object, + }; if outcome.identity.kind != expected_kind || outcome.identity.bucket.as_str() != target.bucket.as_ref() || outcome.identity.object.as_str() != target.object.as_ref() { return None; } - let version_id = target.version_id.filter(|bytes| *bytes != [0; 16]); + let version_id = (!matches!(target.kind, MrfKind::MetadataCorruption)) + .then_some(target.version_id) + .flatten() + .filter(|bytes| *bytes != [0; 16]); let expected_version = version_id.map(|bytes| uuid::Uuid::from_bytes(bytes).to_string()); if outcome.identity.version_id != expected_version { return None; } - let expected_pool = target.scope.and_then(|scope| usize::try_from(scope.pool_index).ok()); - let expected_set = target.scope.and_then(|scope| usize::try_from(scope.set_index).ok()); + let scope = (!matches!(target.kind, MrfKind::MetadataCorruption)) + .then_some(target.scope) + .flatten(); + let expected_pool = scope.and_then(|scope| usize::try_from(scope.pool_index).ok()); + let expected_set = scope.and_then(|scope| usize::try_from(scope.set_index).ok()); if outcome.identity.pool_index != expected_pool || outcome.identity.set_index != expected_set { return None; } @@ -759,7 +814,7 @@ pub(super) fn mrf_verified_repair_event_for_target( bucket: target.bucket.clone(), object: target.object.clone(), version_id, - scope: target.scope, + scope, lease: target.lease, bucket_incarnation_id, disposition, diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index 16e71eb79..6b04bad40 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -26,6 +26,7 @@ use rustfs_madmin::heal_commands::HealResultItem; use std::sync::Mutex as StdMutex; use tempfile::TempDir; +mod root_recovery; mod running_mainline; use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo}; @@ -94,6 +95,7 @@ async fn process_manager_queue_once(manager: &HealManager) { retrying_heals: &manager.retrying_heals, mrf_repair_notice_targets: &manager.mrf_repair_notice_targets, replacement_recovery_anchors: &manager.replacement_recovery_anchors, + root_recovery: &manager.root_recovery, config: &manager.config, statistics: &manager.statistics, storage: &manager.storage, @@ -149,7 +151,7 @@ fn completed_retention_cursor_boundaries_preserve_progress() { #[tokio::test] async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() { - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let request = HealRequest::bucket("bucket".to_string()); manager.insert_task_alias("alias", &request.id).await; let terminal = record_displaced_terminal(&manager.displaced_terminals, &request); @@ -211,7 +213,7 @@ async fn completed_retention_clock_rollback_preserves_terminal_alias_queries() { }, HealTaskStatus::Cancelled, ] { - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let mut snapshot = completed_retention_fixture(completed_at); snapshot.status = status.clone(); let expected_progress = snapshot.progress.clone(); @@ -365,7 +367,7 @@ async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() { use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome}; use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook}; let bucket = "canonical-outcome-cancel-before-finish"; - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let request = HealRequest::object(bucket.to_string(), "object".to_string(), None); let task_id = request.id.clone(); let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None); @@ -416,7 +418,7 @@ async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() { #[tokio::test] async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { let bucket = "completed-retention-retry-cancel"; - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let request = HealRequest::object(bucket.to_string(), "object".to_string(), None); let task_id = request.id.clone(); let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None); @@ -471,7 +473,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han for outcome in ["success", "failed", "cancelled"] { let bucket = format!("completed-retention-{outcome}"); let hook = Arc::new(CompletedRetentionHook::default()); - let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None)); + let manager = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None)); let request = HealRequest::object(bucket.clone(), "object".to_string(), None); let task_id = request.id.clone(); let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None); @@ -711,7 +713,7 @@ impl HealStorageAPI for MockStorage { async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) { for phase in ["listing", "object"] { let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}"); - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let mut request = HealRequest::new( HealType::Prefix { bucket: bucket.clone(), @@ -822,7 +824,7 @@ async fn heal_start_retry_cancellation_preserves_real_executor_progress() { #[tokio::test] async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() { - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( Arc::new(MockStorage), Some(HealConfig { task_timeout: Duration::ZERO, @@ -863,6 +865,88 @@ async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() { COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id); } +#[tokio::test] +async fn heal_start_retry_scheduler_carries_explicit_budget_across_retries() { + let manager = HealManager::new_without_root_recovery_for_test( + Arc::new(MockStorage), + Some(HealConfig { + task_timeout: Duration::ZERO, + event_driven_scheduler_enable: false, + ..Default::default() + }), + ); + let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); + request.source = HealRequestSource::Admin; + request.options.timeout = Some(Duration::from_secs(60)); + let task_id = request.id.clone(); + let created_at = request.created_at; + manager + .submit_heal_request(request) + .await + .expect("admit explicit-budget task"); + + let mut previous_remaining = Duration::from_secs(60); + for expected_attempt in 1..=3 { + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if manager.retrying_heals.lock().await.contains_key(&task_id) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("recoverable executor failure should enter retry backoff"); + + let retry = manager.retrying_heals.lock().await[&task_id].request.clone(); + assert_eq!(retry.id, task_id); + assert_eq!(retry.created_at, created_at); + assert_eq!(retry.source, HealRequestSource::Admin); + assert_eq!(retry.retry_attempts, expected_attempt); + let remaining = retry.options.timeout.expect("retry retains explicit budget"); + assert!( + remaining > Duration::ZERO && remaining < previous_remaining, + "retry attempt {expected_attempt} should carry only the unused explicit budget" + ); + previous_remaining = remaining; + assert_eq!(manager.operations_snapshot().await.queue_length, 0); + assert_eq!(manager.operations_snapshot().await.retrying_tasks, 1); + + let retry_delay = recoverable_heal_retry_delay(expected_attempt); + tokio::time::timeout(retry_delay + Duration::from_secs(1), async { + loop { + if !manager.retrying_heals.lock().await.contains_key(&task_id) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("retry backoff should requeue the same task id"); + assert_eq!(manager.operations_snapshot().await.queue_length, 1); + } + + process_manager_queue_once(&manager).await; + let final_error = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let Ok(HealTaskStatus::Failed { error }) = manager.get_task_status(&task_id).await { + break error; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("retry limit should finish as a terminal failure"); + let final_error_lower = final_error.to_ascii_lowercase(); + assert!( + final_error_lower.contains("insufficient") && final_error_lower.contains("read"), + "terminal failure should retain the recoverable storage error: {final_error}" + ); + assert_eq!(manager.operations_snapshot().await.queue_length, 0); + assert_eq!(manager.operations_snapshot().await.retrying_tasks, 0); +} + struct ManagerRecoveryTestHook { replacement_resume_disk: DiskStore, listed: StdMutex, @@ -962,7 +1046,7 @@ fn scoped_object_request(bucket: &str, object: &str, pool_index: usize, set_inde #[tokio::test] async fn scheduler_bulkhead_starts_other_sets_and_retains_same_set_tail() { - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); { let mut config = manager.config.write().await; config.max_concurrent_heals = 2; @@ -1142,18 +1226,48 @@ fn mrf_verified_repair_event_requires_positive_exact_identity() { assert_eq!(event.bucket_incarnation_id, incarnation); assert_eq!(event.disposition, MrfVerifiedRepairDisposition::Repaired); - assert!( - mrf_verified_repair_event_for_target( - &MrfRepairNoticeTarget { - kind: MrfKind::DecodeFailure, - ..target.clone() - }, - &matching - ) - .is_none(), - "only receipt-producing partial-write object heals can publish verified events today" + let decode_target = MrfRepairNoticeTarget { + kind: MrfKind::DecodeFailure, + ..target.clone() + }; + let decode_outcome = HealObjectOutcome { + identity: HealObjectIdentity { + kind: HealObjectKind::Decode, + ..matching.identity.clone() + }, + ..matching.clone() + }; + let decode_event = + mrf_verified_repair_event_for_target(&decode_target, &decode_outcome).expect("decode repairs publish exact proofs"); + assert_eq!(decode_event.kind, MrfKind::DecodeFailure); + assert_eq!( + decode_event.scope, + Some(MrfScope { + pool_index: 1, + set_index: 2 + }) ); + let metadata_target = MrfRepairNoticeTarget { + kind: MrfKind::MetadataCorruption, + ..target.clone() + }; + let metadata_outcome = HealObjectOutcome { + identity: HealObjectIdentity { + kind: HealObjectKind::Metadata, + version_id: None, + pool_index: None, + set_index: None, + ..matching.identity.clone() + }, + ..matching.clone() + }; + let metadata_event = + mrf_verified_repair_event_for_target(&metadata_target, &metadata_outcome).expect("metadata repairs publish exact proofs"); + assert_eq!(metadata_event.kind, MrfKind::MetadataCorruption); + assert_eq!(metadata_event.version_id, None); + assert_eq!(metadata_event.scope, None); + for rejected in [ HealObjectOutcome { disposition: HealObjectDisposition::Unknown, @@ -1837,7 +1951,7 @@ fn test_heal_request_and_task_metric_labels_match() { #[tokio::test] async fn test_submit_heal_request_returns_merged_for_duplicate() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let request = HealRequest::new( HealType::Object { @@ -1868,7 +1982,7 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() { #[tokio::test] async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None); original.source = HealRequestSource::Admin; let original_id = original.id.clone(); @@ -1895,7 +2009,7 @@ async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() { #[tokio::test] async fn test_task_alias_is_removed_after_terminal_completion() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let original = HealRequest::object("bucket".to_string(), "object".to_string(), None); let original_id = original.id.clone(); let duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None); @@ -1935,7 +2049,7 @@ async fn test_task_alias_is_removed_after_terminal_completion() { #[tokio::test] async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() { let storage: Arc = Arc::new(MockStorage); - let manager = Arc::new(HealManager::new(storage.clone(), None)); + let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None)); let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None); original.source = HealRequestSource::Admin; let original_id = original.id.clone(); @@ -2005,7 +2119,7 @@ async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() { #[tokio::test] async fn test_submit_heal_request_returns_merged_for_active_duplicate() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); + let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None); let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); let active_task = Arc::new(HealTask::from_request(active_request, storage)); manager.active_heals.lock().await.insert(active_task.id.clone(), active_task); @@ -2025,7 +2139,7 @@ async fn test_submit_heal_request_returns_merged_for_active_duplicate() { #[tokio::test] async fn test_active_duplicate_token_can_query_and_cancel_original_task() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); + let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None); let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); let active_task = Arc::new(HealTask::from_request(active_request, storage)); let active_task_id = active_task.id.clone(); @@ -2067,7 +2181,7 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() { #[tokio::test] async fn test_queued_duplicate_token_can_query_and_cancel_original_request() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let original_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); let original_task_id = original_request.id.clone(); let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); @@ -2258,7 +2372,7 @@ fn durable_replacement_recovery_re_admits_only_the_matching_generation() { #[test] fn replacement_recovery_blocker_is_set_scoped() { - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); manager.block_replacement_recovery_set("pool_0_set_0"); @@ -2364,7 +2478,7 @@ async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() { let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone()); let storage = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); + let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None); let mut request = HealRequest::new( HealType::ErasureSet { buckets: vec!["bucket-a".to_string()], @@ -2499,7 +2613,7 @@ fn test_retry_request_for_recoverable_error_stops_at_limit() { } #[tokio::test] -async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() { +async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryable() { let storage: Arc = Arc::new(MockStorage); let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage); let result = Err(task @@ -2513,7 +2627,32 @@ async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted }) .await); - assert!(retry_request_for_result(&task, &result).is_none()); + let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result) + .await + .expect("all-retryable batch failure should rescan within the manager retry budget"); + + assert_eq!(retry_request.id, task.id); + assert_eq!(retry_request.retry_attempts, 1); + assert!(retry_delay > Duration::ZERO); + assert!(error.contains("Lock acquisition timeout")); +} + +#[tokio::test] +async fn test_retry_request_does_not_rescan_batch_with_permanent_failures() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage); + let result = Err(task + .record_batch_failure(BatchHealFailure { + scope: "bucket:bucket".to_string(), + failed: 2, + retryable: 1, + permanent: 1, + first_object: "object-a".to_string(), + first_error: "Lock acquisition timeout".to_string(), + }) + .await); + + assert!(retry_request_for_result_with_budget(&task, &result).await.is_none()); } #[test] @@ -2575,7 +2714,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> #[tokio::test] async fn test_cancel_task_cancels_retrying_backoff() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut request = HealRequest::bucket("bucket".to_string()); request.retry_attempts = 1; let task_id = request.id.clone(); @@ -2602,7 +2741,7 @@ async fn test_cancel_task_cancels_retrying_backoff() { #[tokio::test] async fn test_cancel_tasks_for_path_cancels_retrying_backoff() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut request = HealRequest::bucket("bucket".to_string()); request.retry_attempts = 1; let task_id = request.id.clone(); @@ -2624,7 +2763,7 @@ async fn test_cancel_tasks_for_path_cancels_retrying_backoff() { #[tokio::test] async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); let cluster_request_id = cluster_request.id.clone(); @@ -2663,7 +2802,7 @@ async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() { #[tokio::test] async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); + let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None); let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); let cluster_request_id = cluster_request.id.clone(); @@ -2694,7 +2833,7 @@ async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() { #[tokio::test] async fn test_cancel_tasks_for_empty_path_cancels_retrying_cluster_only() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); cluster_request.retry_attempts = 1; @@ -2733,7 +2872,7 @@ fn test_heal_type_matches_path_accepts_legacy_root() { #[tokio::test] async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut original_request = HealRequest::bucket("bucket".to_string()); original_request.retry_attempts = 1; let original_task_id = original_request.id.clone(); @@ -2769,7 +2908,7 @@ async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() { #[tokio::test] async fn test_get_task_status_reports_pending_for_queued_request() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let request = HealRequest::bucket("bucket".to_string()); let request_id = request.id.clone(); @@ -2793,7 +2932,7 @@ async fn test_get_task_status_reports_pending_for_queued_request() { #[tokio::test] async fn test_operations_snapshot_counts_queue_by_source_and_priority() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut scanner_request = HealRequest::new( HealType::Object { @@ -2850,7 +2989,7 @@ async fn test_operations_snapshot_counts_queue_by_source_and_priority() { // HS-06 (backlog#1870): overlap policy + forceStart semantics. fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager { let storage: Arc = Arc::new(MockStorage); - HealManager::new( + HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { overlap_policy: policy, @@ -2872,6 +3011,12 @@ fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest { request } +fn internal_prefix_request(bucket: &str, prefix: &str) -> HealRequest { + let mut request = admin_prefix_request(bucket, prefix); + request.source = HealRequestSource::Internal; + request +} + async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String { let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); let task_id = task.id.clone(); @@ -2980,7 +3125,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() { #[tokio::test] async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement() { let storage: Arc = Arc::new(MockStorage); - let manager = Arc::new(HealManager::new( + let manager = Arc::new(HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -2988,7 +3133,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement( }), )); - let mut paused = admin_prefix_request("bucket-a", "logs/"); + let mut paused = internal_prefix_request("bucket-a", "logs/"); paused.priority = HealPriority::Low; let hook = Arc::new(DuplicateAdmissionTestHook { request_id: paused.id.clone(), @@ -3018,7 +3163,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement( ); *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None; - let duplicate = admin_prefix_request("bucket-a", "logs/"); + let duplicate = internal_prefix_request("bucket-a", "logs/"); let duplicate_receipt = manager .submit_heal_request_with_receipt(duplicate) .await @@ -3062,7 +3207,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement( #[tokio::test] async fn test_operations_snapshot_counts_active_by_source_and_priority() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut request = HealRequest::bucket("bucket-a".to_string()); request.priority = HealPriority::High; @@ -3084,7 +3229,7 @@ async fn test_operations_snapshot_counts_active_by_source_and_priority() { #[tokio::test] async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut request = HealRequest::bucket("bucket-retry".to_string()); request.priority = HealPriority::Urgent; request.source = HealRequestSource::Admin; @@ -3110,7 +3255,7 @@ async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() { #[tokio::test] async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() { let storage: Arc = Arc::new(MockStorage); - let manager = Arc::new(HealManager::new(storage, None)); + let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage, None)); { let mut config = manager.config.write().await; config.enable_auto_heal = false; @@ -3183,7 +3328,7 @@ async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() { #[tokio::test] async fn test_active_progress_snapshot_sums_active_task_progress() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let first = Arc::new(HealTask::from_request( HealRequest::bucket("bucket-a".to_string()), @@ -3228,7 +3373,7 @@ async fn test_active_progress_snapshot_sums_active_task_progress() { #[tokio::test] async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); manager .submit_heal_request(HealRequest::bucket("bucket".to_string())) @@ -3244,7 +3389,7 @@ async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() #[tokio::test] async fn test_get_task_status_for_path_rejects_token_from_other_active_path() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let bucket_request = HealRequest::bucket("bucket".to_string()); let other_request = HealRequest::bucket("other".to_string()); @@ -3268,7 +3413,7 @@ async fn test_get_task_status_for_path_rejects_token_from_other_active_path() { #[tokio::test] async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let request = HealRequest::bucket("bucket".to_string()); let request_id = request.id.clone(); @@ -3287,7 +3432,7 @@ async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path( #[tokio::test] async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); assert!(matches!( manager.get_task_status_for_path("bucket", "old-token").await, @@ -3298,7 +3443,7 @@ async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() #[tokio::test] async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let request = HealRequest::bucket("bucket".to_string()); let request_id = request.id.clone(); @@ -3321,7 +3466,7 @@ async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() { #[tokio::test] async fn test_get_task_report_queries_queued_task_by_token_without_path() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let request = HealRequest::new( HealType::ErasureSet { @@ -3350,7 +3495,7 @@ async fn test_get_task_report_queries_queued_task_by_token_without_path() { #[tokio::test] async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); // A completed entry recorded in a Retrying state for a task whose // request is also (still) queued under the same id: the retrying @@ -3394,7 +3539,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { #[tokio::test] async fn test_get_task_status_reads_recent_completed_status() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); manager.completed_heals.lock().await.insert( "completed-token".to_string(), @@ -3426,7 +3571,7 @@ async fn test_get_task_status_reads_recent_completed_status() { #[tokio::test] async fn test_get_task_report_for_path_reads_completed_items() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); manager.completed_heals.lock().await.insert( "completed-token".to_string(), @@ -3474,7 +3619,7 @@ async fn test_get_task_report_for_path_reads_completed_items() { #[tokio::test] async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); manager .submit_heal_request(HealRequest::bucket("bucket".to_string())) @@ -3490,7 +3635,7 @@ async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() { #[tokio::test] async fn test_cancel_task_removes_queued_request() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let request = HealRequest::bucket("bucket".to_string()); let request_id = request.id.clone(); @@ -3514,7 +3659,7 @@ async fn mrf_ownership_unverified_completion_does_not_emit_repaired() { let version_id = Some([9u8; 16]); let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket); let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut request = HealRequest::object(bucket.to_string(), object.to_string(), None); request.source = HealRequestSource::Mrf; @@ -3561,7 +3706,7 @@ async fn mrf_ownership_dry_run_and_empty_window_do_not_emit_repaired() { } else { "mrf-dry-run-outcome" }; - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let request = HealRequest::new( if empty_window { HealType::Cluster @@ -3617,7 +3762,7 @@ async fn mrf_ownership_queued_cancel_does_not_emit_repaired() { let object = "object"; let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket); let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let mut request = HealRequest::object(bucket.to_string(), object.to_string(), None); request.source = HealRequestSource::Mrf; @@ -3648,7 +3793,7 @@ async fn mrf_ownership_queued_cancel_does_not_emit_repaired() { #[tokio::test] async fn test_cancel_tasks_for_path_removes_matching_queued_requests() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); let bucket_request = HealRequest::bucket("bucket".to_string()); let bucket_request_id = bucket_request.id.clone(); @@ -3687,7 +3832,7 @@ async fn test_cancel_tasks_for_path_removes_matching_queued_requests() { #[tokio::test] async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -3724,7 +3869,7 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() { #[tokio::test] async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -3767,7 +3912,7 @@ async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() { #[tokio::test] async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -3809,7 +3954,7 @@ async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() { #[tokio::test] async fn test_high_priority_request_displaces_lower_priority_when_queue_full() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -3864,7 +4009,7 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() { #[tokio::test] async fn displaced_task_remains_queryable() { - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( Arc::new(MockStorage), Some(HealConfig { queue_size: 1, @@ -3906,7 +4051,7 @@ async fn displaced_task_remains_queryable() { #[tokio::test] async fn displaced_archive_failure_keeps_queryable_terminal() { - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let mut request = HealRequest::new( HealType::Bucket { bucket: "archive-failure".to_string(), @@ -3929,7 +4074,7 @@ async fn displaced_archive_failure_keeps_queryable_terminal() { #[tokio::test] async fn scheduler_retry_displacement_keeps_evicted_task_queryable() { - let manager = Arc::new(HealManager::new( + let manager = Arc::new(HealManager::new_without_root_recovery_for_test( Arc::new(MockStorage), Some(HealConfig { queue_size: 1, @@ -3994,7 +4139,7 @@ async fn scheduler_retry_displacement_keeps_evicted_task_queryable() { #[tokio::test] async fn concurrent_displacers_produce_one_terminal_generation() { - let manager = Arc::new(HealManager::new( + let manager = Arc::new(HealManager::new_without_root_recovery_for_test( Arc::new(MockStorage), Some(HealConfig { queue_size: 1, @@ -4046,7 +4191,7 @@ async fn concurrent_displacers_produce_one_terminal_generation() { #[tokio::test] async fn successor_chain_is_bounded_and_authorized() { - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( Arc::new(MockStorage), Some(HealConfig { queue_size: 1, @@ -4099,7 +4244,7 @@ async fn successor_chain_is_bounded_and_authorized() { #[tokio::test] async fn displaced_terminal_expires_after_bounded_ttl() { - let manager = HealManager::new(Arc::new(MockStorage), None); + let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None); let mut request = HealRequest::new( HealType::Bucket { bucket: "expires".to_string(), @@ -4122,7 +4267,7 @@ async fn displaced_terminal_expires_after_bounded_ttl() { #[tokio::test] async fn test_displacing_registered_mrf_task_drops_notice_ownership() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -4170,7 +4315,7 @@ async fn test_displacing_registered_mrf_task_drops_notice_ownership() { #[tokio::test] async fn test_submit_heal_request_drops_read_repair_under_pressure() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 10, @@ -4204,7 +4349,7 @@ async fn test_submit_heal_request_drops_read_repair_under_pressure() { #[tokio::test] async fn test_submit_heal_request_drops_low_scanner_under_pressure() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 10, @@ -4238,7 +4383,7 @@ async fn test_submit_heal_request_drops_low_scanner_under_pressure() { #[tokio::test] async fn test_submit_heal_request_accepts_admin_high_under_pressure() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 10, @@ -4278,7 +4423,7 @@ async fn test_mainline_throttle_delays_background_heal_start() { limit: 10, state: AdmissionState::Open, }); - let manager = HealManager::new_with_workload_provider( + let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test( storage, Some(HealConfig { max_concurrent_heals: 1, @@ -4311,7 +4456,7 @@ async fn test_mainline_throttle_delays_background_heal_start_under_write_pressur limit: 10, state: AdmissionState::Open, }); - let manager = HealManager::new_with_workload_provider( + let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test( storage, Some(HealConfig { max_concurrent_heals: 1, @@ -4344,7 +4489,7 @@ async fn test_mainline_throttle_allows_admin_high_start() { limit: 10, state: AdmissionState::Saturated, }); - let manager = HealManager::new_with_workload_provider( + let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test( storage, Some(HealConfig { max_concurrent_heals: 1, @@ -4370,7 +4515,7 @@ async fn test_mainline_throttle_allows_admin_high_start() { #[tokio::test] async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { max_concurrent_heals: 1, @@ -4424,7 +4569,7 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() { #[tokio::test] async fn test_force_start_bypasses_duplicate_and_full_admission() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -4483,7 +4628,7 @@ async fn test_force_start_bypasses_duplicate_and_full_admission() { #[tokio::test] async fn test_force_start_marks_dedup_key_for_future_duplicates() { let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( + let manager = HealManager::new_without_root_recovery_for_test( storage, Some(HealConfig { queue_size: 1, @@ -4537,6 +4682,48 @@ async fn test_force_start_marks_dedup_key_for_future_duplicates() { ); } +#[tokio::test] +async fn same_request_id_replay_reuses_existing_task_without_force_start_duplication() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new_without_root_recovery_for_test(storage, None); + + let mut original = admin_prefix_request("bucket", "logs/"); + original.force_start = true; + let original_id = original.id.clone(); + let accepted = manager + .submit_heal_request_with_receipt(original.clone()) + .await + .expect("original forceStart request should queue"); + assert_eq!(accepted.result, HealAdmissionResult::Accepted); + assert_eq!(accepted.task_id, original_id); + + let replayed = manager + .submit_heal_request_with_receipt(original.clone()) + .await + .expect("same request id and payload should reuse the existing task"); + assert_eq!(replayed.result, HealAdmissionResult::Accepted); + assert_eq!(replayed.task_id, original_id); + assert_eq!( + manager.get_queue_length().await, + 1, + "exact forceStart replay must not create a second queued task" + ); + + let mut changed = original; + changed.options.remove_corrupted = true; + let changed = manager + .submit_heal_request_with_receipt(changed) + .await + .expect("same request id with a changed payload should fail closed"); + assert_eq!(changed.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)); + assert_eq!(changed.task_id, original_id); + assert_eq!( + manager.get_queue_length().await, + 1, + "same-id conflict must not displace or duplicate the original task" + ); +} + #[test] fn test_running_heal_set_counts_groups_set_scoped_tasks() { let storage: Arc = Arc::new(MockStorage); diff --git a/crates/heal/src/heal/manager/tests/root_recovery.rs b/crates/heal/src/heal/manager/tests/root_recovery.rs new file mode 100644 index 000000000..0b3e4e680 --- /dev/null +++ b/crates/heal/src/heal/manager/tests/root_recovery.rs @@ -0,0 +1,1314 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::super::root_recovery::RootHealRecovery; +use super::*; +use crate::heal::RUSTFS_META_BUCKET; +use std::collections::HashSet; + +async fn recovery_disk() -> (TempDir, DiskStore) { + let temp = TempDir::new().expect("temporary root recovery disk"); + let endpoint = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("disk endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("local recovery disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("metadata volume: {error}"), + } + (temp, disk) +} + +fn recovery_manager(disks: Vec) -> HealManager { + let mut manager = HealManager::new( + Arc::new(MockStorage), + Some(HealConfig { + enable_auto_heal: false, + ..Default::default() + }), + ); + manager.root_recovery = Arc::new(RootHealRecovery::with_disks(disks)); + manager +} + +fn root_request() -> HealRequest { + admin_request(HealType::Cluster) +} + +fn admin_request(heal_type: HealType) -> HealRequest { + let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::High); + request.source = HealRequestSource::Admin; + request +} + +fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus { + CompletedHealStatus { + outcome: None, + heal_type: heal_type.clone(), + status: HealTaskStatus::Completed, + progress: Some(HealProgress { + objects_scanned: 1, + objects_healed: 1, + bytes_processed: 64, + ..Default::default() + }), + retained_bytes: std::sync::OnceLock::new(), + result_items_truncated: false, + completed_at, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + } +} + +async fn active_root(manager: &HealManager, request: HealRequest) -> Arc { + let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); + *task.status.write().await = HealTaskStatus::Running; + task.progress.write().await.update_object_progress(1, 1, 0, 0, 128); + manager.active_heals.lock().await.insert(task.id.clone(), task.clone()); + task +} + +#[tokio::test] +async fn root_recovery_shutdown_restart_replays_same_id_and_success_retires_intent() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = root_request(); + request.options.recursive = true; + let task = active_root(&manager, request.clone()).await; + manager.stop().await.expect("durable shutdown handoff"); + assert!(task.cancel_token.is_cancelled()); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay durable root"); + restarted.replay_root_heals().await.expect("replay is idempotent"); + assert_eq!(restarted.get_queue_length().await, 1); + let restored = restarted + .heal_queue + .lock() + .await + .requests() + .next() + .cloned() + .expect("restored request"); + assert_eq!(restored.id, request.id); + assert_eq!(restored.options, request.options); + assert_eq!(restored.priority, request.priority); + assert_eq!(restored.retry_attempts, request.retry_attempts); + assert_eq!(restored.created_at, request.created_at); + + process_manager_queue_once(&restarted).await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if matches!(restarted.get_task_status(&request.id).await, Ok(HealTaskStatus::Completed)) + && !restarted.active_heals.lock().await.contains_key(&request.id) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("restored root executes successfully"); + assert!(restarted.root_recovery.pending().await.expect("read completion").is_empty()); +} + +#[tokio::test] +async fn root_recovery_admin_start_persists_before_shutdown() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = root_request(); + request.options.recursive = true; + let receipt = manager + .submit_heal_request_with_receipt(request.clone()) + .await + .expect("root admission should persist"); + assert_eq!(receipt.result, HealAdmissionResult::Accepted); + assert_eq!(receipt.task_id, request.id); + let pending = manager.root_recovery.pending().await.expect("read durable admission"); + assert_eq!( + pending.iter().map(|request| request.id.as_str()).collect::>(), + [request.id.as_str()] + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay durable admission"); + let queued = restarted + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + assert_eq!(queued, [request.id]); +} + +#[tokio::test] +async fn root_recovery_admin_non_root_types_persist_and_replay() { + for heal_type in [ + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealType::Prefix { + bucket: "bucket".to_string(), + prefix: "logs/2026".to_string(), + }, + HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: Some("version-1".to_string()), + }, + HealType::Metadata { + bucket: "bucket".to_string(), + object: "object".to_string(), + }, + HealType::ECDecode { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + HealType::ErasureSet { + buckets: vec!["bucket".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + ] { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = admin_request(heal_type.clone()); + request.options.recursive = true; + let receipt = manager + .submit_heal_request_with_receipt(request.clone()) + .await + .expect("admin heal admission should persist"); + assert_eq!(receipt.result, HealAdmissionResult::Accepted); + assert_eq!( + manager + .root_recovery + .pending() + .await + .expect("read durable admin state") + .iter() + .map(|request| (&request.id, &request.heal_type)) + .collect::>(), + [(&request.id, &request.heal_type)] + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay durable admin heal"); + let queued = restarted.heal_queue.lock().await.requests().cloned().collect::>(); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].id, request.id); + assert_eq!(queued[0].heal_type, request.heal_type); + assert_eq!(queued[0].options, request.options); + assert_eq!(queued[0].source, HealRequestSource::Admin); + } +} + +#[tokio::test] +async fn root_recovery_non_admin_request_is_not_persisted() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = HealRequest::new( + HealType::Bucket { + bucket: "scanner".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + request.source = HealRequestSource::Scanner; + assert_eq!( + manager + .submit_heal_request(request) + .await + .expect("scanner request should still queue"), + HealAdmissionResult::Accepted + ); + assert!(manager.root_recovery.pending().await.expect("read durable state").is_empty()); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay empty durable state"); + assert_eq!(restarted.get_queue_length().await, 0); +} + +#[tokio::test] +async fn root_recovery_path_cancel_covers_durable_only_non_root_record() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + manager + .root_recovery + .persist(&request) + .await + .expect("durable bucket responsibility"); + assert_eq!( + manager + .cancel_tasks_for_path("bucket") + .await + .expect("cancel durable-only bucket path"), + 1 + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted + .replay_root_heals() + .await + .expect("restart after durable path cancellation"); + assert_eq!(restarted.get_queue_length().await, 0); + assert_eq!( + restarted + .get_task_status_for_path("bucket", &request.id) + .await + .expect("durable cancellation remains queryable by path"), + HealTaskStatus::Cancelled + ); + assert_eq!( + restarted + .get_task_status(&request.id) + .await + .expect("durable cancellation remains queryable by id"), + HealTaskStatus::Cancelled + ); +} + +#[tokio::test] +async fn root_recovery_active_cancel_is_queryable_after_restart_for_scoped_admin() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + active_root(&manager, request.clone()).await; + manager + .root_recovery + .persist(&request) + .await + .expect("durable active bucket responsibility"); + + manager.cancel_task(&request.id).await.expect("cancel active bucket"); + assert!( + manager + .root_recovery + .pending() + .await + .expect("active terminal retires intent") + .is_empty() + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted + .replay_root_heals() + .await + .expect("restart after active cancellation"); + assert_eq!(restarted.get_queue_length().await, 0); + assert_eq!( + restarted + .get_task_status(&request.id) + .await + .expect("active cancellation remains queryable by id"), + HealTaskStatus::Cancelled + ); + assert_eq!( + restarted + .get_task_status_for_path("bucket", &request.id) + .await + .expect("active cancellation remains queryable by path"), + HealTaskStatus::Cancelled + ); +} + +#[tokio::test] +async fn root_recovery_terminal_receipt_wins_over_stale_pending_scoped_intent_after_restart() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + manager + .root_recovery + .persist(&request) + .await + .expect("durable bucket responsibility"); + manager + .publish_admin_cancelled_terminal(&request.id, &request.heal_type, request.source) + .await + .expect("publish terminal receipt"); + manager + .root_recovery + .persist(&request) + .await + .expect("restore stale pending intent after terminal publication"); + assert!( + manager + .root_recovery + .pending() + .await + .expect("terminal masks stale pending") + .is_empty() + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted + .replay_root_heals() + .await + .expect("restart with terminal and stale pending"); + assert_eq!(restarted.get_queue_length().await, 0); + assert_eq!( + restarted + .get_task_status(&request.id) + .await + .expect("terminal status survives stale pending"), + HealTaskStatus::Cancelled + ); +} + +#[tokio::test] +async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + manager + .root_recovery + .persist(&request) + .await + .expect("durable bucket responsibility"); + let completed = CompletedHealStatus { + outcome: None, + heal_type: request.heal_type.clone(), + status: HealTaskStatus::Completed, + progress: Some(HealProgress { + objects_scanned: 2, + objects_healed: 2, + bytes_processed: 128, + ..Default::default() + }), + retained_bytes: std::sync::OnceLock::new(), + result_items_truncated: false, + completed_at: SystemTime::now(), + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + }; + assert!( + manager + .publish_admin_terminal(&request.id, &request.heal_type, request.source, &completed) + .await + .expect("publish completed terminal") + ); + assert!( + manager + .root_recovery + .pending() + .await + .expect("completed terminal retires intent") + .is_empty() + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("restart after completed terminal"); + assert_eq!(restarted.get_queue_length().await, 0); + assert_eq!( + restarted + .get_task_status_for_path("bucket", &request.id) + .await + .expect("completed terminal remains queryable by path"), + HealTaskStatus::Completed + ); + let progress = restarted + .get_task_progress(&request.id) + .await + .expect("completed terminal exposes progress"); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 2); +} + +#[tokio::test] +async fn root_recovery_terminal_receipt_ttl_boundary_matches_completed_status_retention() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + let completed_at = SystemTime::now(); + let completed = completed_admin_status(&request.heal_type, completed_at); + assert!( + manager + .publish_admin_terminal(&request.id, &request.heal_type, request.source, &completed) + .await + .expect("publish terminal receipt") + ); + + let boundary = completed_at + KEEP_HEAL_TASK_STATUS_DURATION; + assert_eq!( + manager + .root_recovery + .completed(&request.id) + .await + .expect("read retained terminal") + .expect("terminal retained at exact TTL boundary") + .status, + HealTaskStatus::Completed + ); + let report = manager + .root_recovery + .gc_terminal_receipts_once(boundary) + .await + .expect("boundary GC"); + assert_eq!(report.terminals_removed, 0); + assert_eq!( + manager + .root_recovery + .completed(&request.id) + .await + .expect("read retained terminal") + .expect("terminal retained before wall-clock advances") + .status, + HealTaskStatus::Completed + ); + + let expired = boundary + Duration::from_nanos(1); + let report = manager + .root_recovery + .gc_terminal_receipts_once(expired) + .await + .expect("expired GC"); + assert_eq!(report.terminals_removed, 1); + assert!(matches!(manager.get_task_status(&request.id).await, Err(Error::TaskNotFound { .. }))); +} + +#[tokio::test] +async fn root_recovery_terminal_gc_removes_stale_pending_before_expired_receipt() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + manager + .root_recovery + .persist(&request) + .await + .expect("durable bucket responsibility"); + let now = SystemTime::now(); + let completed = completed_admin_status(&request.heal_type, now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1)); + assert!( + manager + .publish_admin_terminal(&request.id, &request.heal_type, request.source, &completed) + .await + .expect("publish expired terminal receipt") + ); + manager + .root_recovery + .persist(&request) + .await + .expect("recreate stale pending intent after terminal publication"); + assert!( + manager + .root_recovery + .pending() + .await + .expect("terminal still masks stale pending") + .is_empty(), + "an expired receipt must continue masking stale pending until GC retires the pending owner" + ); + assert!(matches!(manager.get_task_status(&request.id).await, Err(Error::TaskNotFound { .. }))); + + let first = manager + .root_recovery + .gc_terminal_receipts_once(now) + .await + .expect("first GC removes stale pending only"); + assert_eq!(first.pending_removed, 1); + assert_eq!(first.terminals_removed, 0); + assert!( + disk.read_all(RUSTFS_META_BUCKET, &format!("terminal-root-heal-{}.json", request.id)) + .await + .is_ok() + ); + assert!(manager.root_recovery.pending().await.expect("pending retired").is_empty()); + + let second = manager + .root_recovery + .gc_terminal_receipts_once(now) + .await + .expect("second GC removes unneeded expired terminal"); + assert_eq!(second.pending_removed, 0); + assert_eq!(second.terminals_removed, 1); + assert!( + disk.read_all(RUSTFS_META_BUCKET, &format!("terminal-root-heal-{}.json", request.id)) + .await + .is_err() + ); +} + +#[tokio::test] +async fn root_recovery_terminal_gc_is_delete_budget_bounded() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let now = SystemTime::now(); + let expired_at = now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1); + for _ in 0..=64 { + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + let completed = completed_admin_status(&request.heal_type, expired_at); + manager + .publish_admin_terminal(&request.id, &request.heal_type, request.source, &completed) + .await + .expect("publish expired terminal receipt"); + } + + let report = manager + .root_recovery + .gc_terminal_receipts_once(now) + .await + .expect("budgeted terminal GC"); + assert_eq!(report.terminals_removed, 64); + assert!(report.budget_exhausted); + let terminal_entries = disk + .list_dir("", RUSTFS_META_BUCKET, "", -1) + .await + .expect("list remaining terminal receipts") + .into_iter() + .filter(|entry| entry.starts_with("terminal-root-heal-")) + .count(); + assert_eq!(terminal_entries, 1); +} + +#[tokio::test] +async fn root_recovery_corrupt_terminal_receipt_retains_pending_fail_closed() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + manager + .root_recovery + .persist(&request) + .await + .expect("durable bucket responsibility"); + let completed = completed_admin_status( + &request.heal_type, + SystemTime::now() - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1), + ); + manager + .publish_admin_terminal(&request.id, &request.heal_type, request.source, &completed) + .await + .expect("publish terminal receipt"); + manager + .root_recovery + .persist(&request) + .await + .expect("restore stale pending intent"); + disk.write_all( + RUSTFS_META_BUCKET, + &format!("terminal-root-heal-{}.json", request.id), + br#"{"schema":1,"task_id":"not-the-same-id"}"#.to_vec().into(), + ) + .await + .expect("corrupt terminal receipt"); + + assert!( + manager + .root_recovery + .gc_terminal_receipts_once(SystemTime::now()) + .await + .is_err(), + "corrupt terminal receipt must fail closed" + ); + assert!( + disk.read_all(RUSTFS_META_BUCKET, &format!("root-heal-{}.json", request.id)) + .await + .is_ok() + ); + assert!(manager.root_recovery.pending().await.is_err()); +} + +#[tokio::test] +async fn root_recovery_admin_start_fails_closed_when_owner_is_unavailable() { + let (_temp, disk) = recovery_disk().await; + let (unavailable_temp, unavailable) = recovery_disk().await; + std::fs::remove_dir_all(unavailable_temp.path().join(RUSTFS_META_BUCKET)).expect("make owner volume unavailable"); + let manager = recovery_manager(vec![unavailable, disk.clone()]); + let request = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + + assert!( + manager.submit_heal_request_with_receipt(request).await.is_err(), + "admin admission must fail closed when the durable owner cannot be checked" + ); + assert_eq!(manager.get_queue_length().await, 0); + assert!( + RootHealRecovery::with_disks(vec![disk]) + .pending() + .await + .expect("other disk remains empty") + .is_empty() + ); +} + +#[tokio::test] +async fn root_recovery_force_start_cancels_only_overlapping_durable_admin_records() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let old = admin_request(HealType::Bucket { + bucket: "bucket-a".to_string(), + }); + let disjoint = admin_request(HealType::Bucket { + bucket: "bucket-b".to_string(), + }); + manager.root_recovery.persist(&old).await.expect("old bucket owner"); + manager.root_recovery.persist(&disjoint).await.expect("disjoint bucket owner"); + + let mut replacement = admin_request(HealType::Prefix { + bucket: "bucket-a".to_string(), + prefix: "logs/".to_string(), + }); + replacement.force_start = true; + assert_eq!( + manager + .submit_heal_request(replacement.clone()) + .await + .expect("forceStart should replace only the overlapping durable owner"), + HealAdmissionResult::Accepted + ); + + let mut pending = manager + .root_recovery + .pending() + .await + .expect("read durable owners") + .into_iter() + .map(|request| (request.id, request.heal_type)) + .collect::>(); + pending.sort_by(|left, right| left.0.cmp(&right.0)); + let mut expected = vec![ + (disjoint.id.clone(), disjoint.heal_type.clone()), + (replacement.id.clone(), replacement.heal_type.clone()), + ]; + expected.sort_by(|left, right| left.0.cmp(&right.0)); + assert_eq!(pending, expected); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay surviving owners"); + let queued_ids = restarted + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + assert_eq!(queued_ids, HashSet::from([disjoint.id, replacement.id])); +} + +#[tokio::test] +async fn root_recovery_queued_non_root_admin_owner_is_not_priority_displaced() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + manager.config.write().await.queue_size = 1; + let mut durable = admin_request(HealType::Bucket { + bucket: "bucket".to_string(), + }); + durable.priority = HealPriority::Low; + assert_eq!( + manager + .submit_heal_request(durable.clone()) + .await + .expect("low-priority admin bucket should queue durably"), + HealAdmissionResult::Accepted + ); + + let mut urgent = HealRequest::new( + HealType::Object { + bucket: "other".to_string(), + object: "object".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Urgent, + ); + urgent.source = HealRequestSource::Internal; + assert_eq!( + manager + .submit_heal_request(urgent) + .await + .expect("durable admin owner cannot be displaced"), + HealAdmissionResult::Full + ); + assert_eq!( + manager + .root_recovery + .pending() + .await + .expect("read durable bucket") + .iter() + .map(|request| request.id.as_str()) + .collect::>(), + [durable.id.as_str()] + ); +} + +#[tokio::test] +async fn root_recovery_legacy_schema_replays_as_cluster() { + #[derive(serde::Serialize)] + struct LegacyRootHealIntent<'a> { + schema: u32, + task_id: &'a str, + options: &'a HealOptions, + priority: HealPriority, + retry_attempts: u32, + created_at: SystemTime, + } + + let (_temp, disk) = recovery_disk().await; + let request = root_request(); + let path = format!("root-heal-{}.json", request.id); + let bytes = serde_json::to_vec(&LegacyRootHealIntent { + schema: 1, + task_id: &request.id, + options: &request.options, + priority: request.priority, + retry_attempts: request.retry_attempts, + created_at: request.created_at, + }) + .expect("legacy root recovery JSON"); + disk.write_all(RUSTFS_META_BUCKET, &path, bytes.into()) + .await + .expect("write legacy root record"); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay legacy root record"); + let queued = restarted.heal_queue.lock().await.requests().cloned().collect::>(); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].id, request.id); + assert_eq!(queued[0].heal_type, HealType::Cluster); +} + +#[tokio::test] +async fn root_recovery_rejected_admin_start_does_not_persist() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + manager.config.write().await.queue_size = 0; + let request = root_request(); + + let receipt = manager + .submit_heal_request_with_receipt(request.clone()) + .await + .expect("full admission reports a receipt"); + assert_eq!(receipt.result, HealAdmissionResult::Full); + assert!(manager.root_recovery.pending().await.expect("read durable state").is_empty()); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("replay empty durable state"); + assert_eq!(restarted.get_queue_length().await, 0); +} + +#[tokio::test] +async fn root_recovery_queued_owner_is_not_priority_displaced() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + manager.config.write().await.queue_size = 1; + let mut root = root_request(); + root.priority = HealPriority::Low; + assert_eq!( + manager + .submit_heal_request(root.clone()) + .await + .expect("low-priority root should queue"), + HealAdmissionResult::Accepted + ); + + let mut bucket = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Urgent, + ); + bucket.source = HealRequestSource::Admin; + assert_eq!( + manager + .submit_heal_request(bucket) + .await + .expect("durable root owner cannot be displaced"), + HealAdmissionResult::Full + ); + let pending = manager.root_recovery.pending().await.expect("read durable root"); + assert_eq!(pending.iter().map(|request| request.id.as_str()).collect::>(), [root.id.as_str()]); + let queued = manager + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + assert_eq!(queued, [root.id]); +} + +#[tokio::test] +async fn root_recovery_explicit_cancel_covers_active_queued_retrying_and_durable_only() { + for state in ["active", "queued", "retrying", "durable_only", "root_path"] { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let request = root_request(); + manager.root_recovery.persist(&request).await.expect("durable responsibility"); + match state { + "active" => { + active_root(&manager, request.clone()).await; + } + "queued" => { + manager.replay_root_heals().await.expect("queued recovery"); + } + "retrying" => { + insert_retrying_request(&manager, request.clone()).await; + } + _ => {} + } + if state == "root_path" { + assert_eq!(manager.cancel_tasks_for_path("").await.expect("cancel durable root path"), 1); + } else { + manager.cancel_task(&request.id).await.expect("cancel root responsibility"); + } + drop(manager); + let restarted = recovery_manager(vec![disk]); + restarted + .replay_root_heals() + .await + .expect("restart after explicit cancellation"); + assert_eq!(restarted.get_queue_length().await, 0, "state={state}"); + } +} + +#[tokio::test] +async fn root_recovery_force_start_cancels_durable_only_responsibility() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let old = root_request(); + manager + .root_recovery + .persist(&old) + .await + .expect("old terminal responsibility"); + let mut new = root_request(); + new.force_start = true; + assert_eq!( + manager + .submit_heal_request(new.clone()) + .await + .expect("force start replacement"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .root_recovery + .pending() + .await + .expect("new owner retained") + .iter() + .map(|request| request.id.as_str()) + .collect::>(), + [new.id.as_str()] + ); + manager.stop().await.expect("persist new root only"); + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("restart replacement"); + let ids = restarted + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + assert_eq!(ids, [new.id]); +} + +#[tokio::test] +async fn root_recovery_force_start_preserves_disjoint_durable_only_admin_work() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let old_overlap = admin_request(HealType::Bucket { + bucket: "overlap".to_string(), + }); + let old_disjoint = admin_request(HealType::Bucket { + bucket: "disjoint".to_string(), + }); + manager + .root_recovery + .persist(&old_overlap) + .await + .expect("durable overlapping bucket"); + manager + .root_recovery + .persist(&old_disjoint) + .await + .expect("durable disjoint bucket"); + + let mut replacement = admin_request(HealType::Bucket { + bucket: "overlap".to_string(), + }); + replacement.force_start = true; + assert_eq!( + manager + .submit_heal_request(replacement.clone()) + .await + .expect("forceStart replaces overlapping durable owner"), + HealAdmissionResult::Accepted + ); + drop(manager); + + let restarted = recovery_manager(vec![disk]); + restarted + .replay_root_heals() + .await + .expect("restart after selective forceStart"); + let mut ids = restarted + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + ids.sort(); + let mut expected = vec![old_disjoint.id, replacement.id]; + expected.sort(); + assert_eq!(ids, expected); +} + +#[tokio::test] +async fn root_recovery_force_start_replaces_fresh_queued_and_retrying_admin_roots() { + for retrying in [false, true] { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let old = root_request(); + if retrying { + insert_retrying_request(&manager, old.clone()).await; + } else { + manager.submit_heal_request(old.clone()).await.expect("queue original root"); + } + if retrying { + assert!( + manager + .root_recovery + .pending() + .await + .expect("retrying not handed off yet") + .is_empty() + ); + } else { + assert_eq!( + manager + .root_recovery + .pending() + .await + .expect("queued root is durable immediately") + .iter() + .map(|request| request.id.as_str()) + .collect::>(), + [old.id.as_str()] + ); + } + let mut new = root_request(); + new.force_start = true; + assert_eq!( + manager.submit_heal_request(new.clone()).await.expect("force replacement"), + HealAdmissionResult::Accepted + ); + manager.stop().await.expect("handoff only the new responsibility"); + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("restart after forceStart"); + let ids = restarted + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + assert_eq!(ids, [new.id], "retrying={retrying}; old={}", old.id); + } +} + +#[tokio::test] +async fn root_recovery_invalid_records_are_retained_without_partial_replay() { + for kind in ["truncated", "schema", "identity", "option", "no_lock"] { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let valid = root_request(); + let invalid = root_request(); + manager.root_recovery.persist(&valid).await.expect("valid root record"); + manager + .root_recovery + .persist(&invalid) + .await + .expect("record before corruption"); + let path = format!("root-heal-{}.json", invalid.id); + let original = disk.read_all(RUSTFS_META_BUCKET, &path).await.expect("read root record"); + let mut value: serde_json::Value = serde_json::from_slice(&original).expect("record JSON"); + match kind { + "schema" => value["schema"] = 3.into(), + "identity" => value["task_id"] = valid.id.clone().into(), + "option" => value["options"]["future_delete_mode"] = true.into(), + "no_lock" => value["options"]["no_lock"] = true.into(), + _ => {} + } + let bytes = if kind == "truncated" { + b"{".to_vec() + } else { + serde_json::to_vec(&value).expect("modified record") + }; + disk.write_all(RUSTFS_META_BUCKET, &path, bytes.clone().into()) + .await + .expect("inject bad record"); + assert!(manager.replay_root_heals().await.is_err(), "kind={kind}"); + assert_eq!(manager.get_queue_length().await, 0, "no partial admission for {kind}"); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, &path) + .await + .expect("bad record retained") + .as_ref(), + bytes + ); + let mut forced = root_request(); + forced.force_start = true; + assert!( + manager.submit_heal_request(forced).await.is_err(), + "forceStart must not discard unknown state" + ); + } +} + +#[tokio::test] +async fn root_recovery_failed_handoff_keeps_runtime_owner_and_does_not_try_another_disk() { + let (_temp, disk) = recovery_disk().await; + let (unavailable_temp, unavailable) = recovery_disk().await; + std::fs::remove_dir_all(unavailable_temp.path().join(RUSTFS_META_BUCKET)).expect("make owner volume unavailable"); + let manager = recovery_manager(vec![unavailable, disk.clone()]); + let task = active_root(&manager, root_request()).await; + assert!(manager.stop().await.is_err()); + assert!( + manager.cancel_task(&task.id).await.is_err(), + "missing owner cannot acknowledge cancellation" + ); + assert!(!manager.cancel_token.is_cancelled()); + assert!(!task.cancel_token.is_cancelled()); + assert!(manager.active_heals.lock().await.contains_key(&task.id)); + assert!( + RootHealRecovery::with_disks(vec![disk]) + .pending() + .await + .expect("other disk remains empty") + .is_empty() + ); +} + +#[tokio::test] +async fn root_recovery_shutdown_fences_new_admission_and_preserves_later_cancellation() { + for operation_kind in ["submit", "force_start", "cancel"] { + let cancel = operation_kind == "cancel"; + let (_temp, disk) = recovery_disk().await; + let manager = Arc::new(recovery_manager(vec![disk.clone()])); + let request = root_request(); + active_root(&manager, request.clone()).await; + let queue = manager.heal_queue.lock().await; + let stopping = manager.clone(); + let stop = tokio::spawn(async move { stopping.stop().await }); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if manager.active_heals.try_lock().is_err() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown owns active lock while waiting for queue"); + let concurrent = manager.clone(); + let operation = tokio::spawn(async move { + if cancel { + concurrent.cancel_task(&request.id).await + } else { + let mut new = root_request(); + new.force_start = operation_kind == "force_start"; + concurrent.submit_heal_request(new).await.map(|_| ()) + } + }); + drop(queue); + stop.await.expect("shutdown task").expect("durable shutdown"); + let result = operation.await.expect("concurrent operation"); + assert_eq!(result.is_ok(), cancel, "operation={operation_kind}"); + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("read final responsibility"); + assert_eq!(restarted.get_queue_length().await, usize::from(!cancel)); + } +} + +#[tokio::test] +async fn root_recovery_exhausted_timeout_is_not_reset_by_restart() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = root_request(); + request.options.timeout = Some(Duration::from_secs(10)); + let task = active_root(&manager, request.clone()).await; + task.set_execution_elapsed_for_test(Duration::from_secs(11)).await; + manager.stop().await.expect("persist exhausted execution budget"); + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("restore bounded request"); + assert_eq!( + restarted + .heal_queue + .lock() + .await + .requests() + .next() + .expect("restored root") + .options + .timeout, + Some(Duration::ZERO) + ); + process_manager_queue_once(&restarted).await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if matches!(restarted.get_task_status(&request.id).await, Ok(HealTaskStatus::Timeout)) { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("exhausted request stays timed out"); + restarted + .cancel_task(&request.id) + .await + .expect("timeout responsibility remains cancellable"); + assert!(restarted.root_recovery.pending().await.expect("retired timeout").is_empty()); +} + +#[tokio::test] +async fn root_recovery_shutdown_preserves_remaining_execution_budget() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = root_request(); + request.options.timeout = Some(Duration::from_secs(60)); + let task = active_root(&manager, request).await; + task.set_execution_elapsed_for_test(Duration::from_secs(20)).await; + manager.stop().await.expect("handoff with consumed execution time"); + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("restore remaining budget"); + let queue = restarted.heal_queue.lock().await; + let remaining = queue + .requests() + .next() + .expect("restored root") + .options + .timeout + .expect("remaining timeout"); + assert!(remaining <= Duration::from_secs(40), "elapsed execution must not be refunded"); + assert!( + remaining >= Duration::from_secs(30), + "shutdown fixture should retain most of its remaining budget" + ); +} + +#[tokio::test] +async fn root_recovery_force_start_after_shutdown_does_not_retire_original_owner() { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let old = root_request(); + active_root(&manager, old.clone()).await; + manager.stop().await.expect("handoff original root"); + let mut new = root_request(); + new.force_start = true; + assert!(manager.submit_heal_request(new).await.is_err()); + let restarted = recovery_manager(vec![disk]); + restarted.replay_root_heals().await.expect("original responsibility remains"); + let ids = restarted + .heal_queue + .lock() + .await + .requests() + .map(|request| request.id.clone()) + .collect::>(); + assert_eq!(ids, [old.id]); +} + +#[tokio::test] +async fn root_recovery_terminal_timeout_updates_only_existing_journal_before_second_restart() { + for durable in [false, true] { + let (_temp, disk) = recovery_disk().await; + let manager = recovery_manager(vec![disk.clone()]); + let mut request = root_request(); + request.options.timeout = Some(Duration::from_nanos(1)); + if durable { + manager + .root_recovery + .persist(&request) + .await + .expect("persist nonzero execution budget"); + manager.replay_root_heals().await.expect("first restart"); + } else { + manager + .submit_heal_request(request.clone()) + .await + .expect("first root execution"); + } + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if matches!(manager.get_task_status(&request.id).await, Ok(HealTaskStatus::Timeout)) + && !manager.active_heals.lock().await.contains_key(&request.id) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("real execution exhausts a nonzero budget"); + assert!(!manager.active_heals.lock().await.contains_key(&request.id)); + let restarted = recovery_manager(vec![disk]); + restarted + .replay_root_heals() + .await + .expect("second restart after terminal timeout"); + let queue = restarted.heal_queue.lock().await; + assert_eq!( + queue + .requests() + .next() + .expect("remaining timeout responsibility") + .options + .timeout, + Some(Duration::ZERO), + "durable={durable}" + ); + } +} diff --git a/crates/heal/src/heal/manager/tests/running_mainline.rs b/crates/heal/src/heal/manager/tests/running_mainline.rs index 4f21d7238..f26e4bece 100644 --- a/crates/heal/src/heal/manager/tests/running_mainline.rs +++ b/crates/heal/src/heal/manager/tests/running_mainline.rs @@ -141,7 +141,7 @@ async fn start_fixture( started: AtomicUsize::new(0), committed: AtomicUsize::new(0), }); - let manager = HealManager::new_with_workload_provider( + let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test( storage.clone(), Some(HealConfig { mainline_throttle_enable: pacing_enabled, diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index e7885fb91..92a806f19 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -27,10 +27,10 @@ pub mod task; pub mod utils; use storage_api::owner::{ - ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, - EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskOption, - EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, - ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk, + ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME, + ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, + EcstoreDiskError, EcstoreDiskOption, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, + EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk, }; pub use erasure_healer::ErasureSetHealer; @@ -41,6 +41,7 @@ pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType}; pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME; pub(crate) const BUCKET_META_PREFIX: &str = ECSTORE_BUCKET_META_PREFIX; pub(crate) const RUSTFS_META_BUCKET: &str = ECSTORE_RUSTFS_META_BUCKET; +pub(crate) const POOL_META_NAME: &str = ECSTORE_POOL_META_NAME; /// Marker written to every local disk while the process runs; removed by /// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs index 95c66a23c..54be65728 100644 --- a/crates/heal/src/heal/mrf_queue.rs +++ b/crates/heal/src/heal/mrf_queue.rs @@ -36,7 +36,7 @@ use super::{DiskStore, HealDiskExt as _, local_disk_map_read}; use crate::heal::manager::{HealManager, MrfRepairNoticeTarget}; use metrics::{counter, gauge}; -use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIngressResult, MrfIntent}; +use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfDurableRepairAnchor, MrfIngressResult, MrfIntent}; use rustfs_heal_contracts::heal_channel::{HealAdmissionDropReason, HealAdmissionResult}; use std::collections::{HashSet, VecDeque}; use std::sync::Arc; @@ -294,7 +294,11 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { }; let attempts = data[3]; let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?); - let has_version = data[12] != 0; + let has_version = match data[12] { + 0 => false, + 1 => true, + _ => return None, + }; let mut cursor = MRF_RECORD_FIXED_HEAD; let version_id = if has_version { if data.len() < cursor + 16 { @@ -517,6 +521,8 @@ async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> c struct MrfRuntime { queue: MrfQueue, config: MrfConsumerConfig, + checkpoint_owner: Uuid, + next_checkpoint_sequence: u64, new_since_flush: usize, /// True while the in-memory pending set has changed since the last /// journal flush (push, pop, or an attempts bump that alters the encoded @@ -527,6 +533,18 @@ struct MrfRuntime { /// True while a journal snapshot exists on disk that may still be needed /// for replay or cleanup. journal_on_disk: bool, + /// True when replay observed a responsibility that cannot be discharged by + /// a complete verified repair proof in this process. + retain_replay_journal: bool, + /// Partial-write responsibilities accepted from replay and waiting for an + /// exact storage-owned proof before the startup journal can be deleted. + durable_replay_anchors: Vec, + /// Startup replay source to remove after the retained replay + /// responsibilities are discharged. `None` means the runtime only needs + /// the legacy journal cleanup path for snapshots it wrote itself. + replay_cleanup: Option, + /// Last committed checkpoint published by this runtime flush path. + runtime_checkpoint: Option<(Uuid, u64)>, /// Earliest instant a full-admission retry may proceed. backoff_until: Option, } @@ -550,6 +568,34 @@ impl MrfRuntime { async fn flush(&mut self) { let (authoritative, legacy) = self.snapshot(); + let (committed_persisted, committed_on_disk) = if authoritative.is_empty() { + (true, false) + } else { + match snapshot::publish_committed_snapshot( + &journal_disks().await, + self.checkpoint_owner, + self.next_checkpoint_sequence, + &authoritative, + self.config.journal_max_bytes, + ) + .await + { + Ok(publication) => { + self.runtime_checkpoint = Some((publication.owner, publication.sequence)); + self.next_checkpoint_sequence = publication.sequence.saturating_add(1); + (true, true) + } + Err(err) => { + tracing::warn!( + target: "rustfs::heal::mrf", + error = %err, + sequence = self.next_checkpoint_sequence, + "MRF committed checkpoint publish failed; retaining previous replay anchor" + ); + (false, false) + } + } + }; let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await; if !authoritative.is_empty() { counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); @@ -560,10 +606,10 @@ impl MrfRuntime { // old reader from observing a newer epoch that a new reader cannot // see when the canonical write is unavailable. let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await; - // Keep dirty until both the authoritative snapshot and its - // compatibility mirror have been accepted; otherwise a one-sided - // failure would never retry the missing file. - let persisted = authoritative_persisted && legacy_persisted; + // Keep dirty until the committed checkpoint, authoritative snapshot, + // and compatibility mirror have all been accepted; otherwise a + // one-sided failure would never retry the missing recovery anchor. + let persisted = committed_persisted && authoritative_persisted && legacy_persisted; self.new_since_flush = 0; // Keep the dirty flag when every disk write failed: a clean backlog // would otherwise never rewrite, losing the periodic persist retry a @@ -571,7 +617,7 @@ impl MrfRuntime { if persisted { self.dirty = false; } - self.journal_on_disk |= authoritative_persisted || legacy_persisted; + self.journal_on_disk |= committed_on_disk || authoritative_persisted || legacy_persisted; } /// Drain pending intents into the heal manager until it is full, the @@ -624,6 +670,68 @@ impl MrfRuntime { gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth())); gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes())); } + + fn retained_replay_journal(&self) -> bool { + self.retain_replay_journal || !self.durable_replay_anchors.is_empty() + } + + fn replay_cleanup_to_delete(&self) -> Option { + if self.journal_on_disk && !self.retained_replay_journal() { + Some(self.replay_cleanup.unwrap_or(ReplayCleanup::Legacy)) + } else { + None + } + } + + async fn delete_idle_recovery_anchors(&mut self) -> bool { + let runtime_deleted = match self.runtime_checkpoint { + Some((owner, sequence)) => { + match snapshot::delete_committed_snapshots_through(owner, sequence, self.config.journal_max_bytes).await { + Ok(deleted) => deleted, + Err(err) => { + tracing::warn!( + target: "rustfs::heal::mrf", + error = %err, + sequence, + "MRF runtime checkpoint cleanup failed" + ); + false + } + } + } + None => true, + }; + let replay_deleted = match self.replay_cleanup_to_delete() { + Some(cleanup) => delete_replay_source(cleanup, self.config.journal_max_bytes).await, + None => true, + }; + if runtime_deleted && replay_deleted { + self.runtime_checkpoint = None; + self.replay_cleanup = None; + true + } else { + false + } + } + + fn discharge_durable_replay_anchors(&mut self) { + if self.durable_replay_anchors.is_empty() { + return; + } + let mut buckets: Vec> = self + .durable_replay_anchors + .iter() + .map(|anchor| anchor.bucket.clone()) + .collect(); + buckets.sort_unstable(); + buckets.dedup(); + for bucket in buckets { + rustfs_common::mrf_channel::consume_recorded_verified_mrf_repair_events_for( + bucket.as_ref(), + &mut self.durable_replay_anchors, + ); + } + } } /// Initialize the global MRF channel (honoring `RUSTFS_HEAL_MRF_ENABLE`) and @@ -673,10 +781,77 @@ pub async fn replay_journal_once(manager: &Arc) -> usize { struct ReplayOutcome { replayed: usize, journal_on_disk: bool, + retain_journal_for_replay: bool, + durable_replay_anchors: Vec, + cleanup: Option, + next_checkpoint_sequence: u64, } -fn replay_must_retain_journal(rearm_incomplete: bool, pending_depth: usize) -> bool { - rearm_incomplete || pending_depth > 0 +fn replay_must_retain_journal( + rearm_incomplete: bool, + pending_depth: usize, + accepted_without_durable_anchor: bool, + durable_replay_anchors: usize, +) -> bool { + rearm_incomplete || pending_depth > 0 || accepted_without_durable_anchor || durable_replay_anchors > 0 +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ReplayCleanup { + Legacy, + Committed { owner: Uuid, sequence: u64 }, +} + +struct ReplaySource { + data: Vec, + cleanup: ReplayCleanup, +} + +async fn read_replay_source(max_bytes: usize) -> Result, snapshot::SnapshotError> { + if let Some(committed) = snapshot::inspect_local_committed_snapshot(max_bytes).await? { + return Ok(Some(ReplaySource { + data: committed.payload().to_vec(), + cleanup: ReplayCleanup::Committed { + owner: committed.owner(), + sequence: committed.sequence(), + }, + })); + } + // The scoped file is a complete authoritative legacy snapshot. Fall back + // to the v1 mirror only when the authoritative path is unavailable; + // merging both files could combine records from different flush epochs. + let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await { + Some(data) => data, + None => match read_journal(MRF_JOURNAL_PATH).await { + Some(data) => data, + None => return Ok(None), + }, + }; + Ok(Some(ReplaySource { + data, + cleanup: ReplayCleanup::Legacy, + })) +} + +async fn delete_replay_source(cleanup: ReplayCleanup, max_bytes: usize) -> bool { + let committed_deleted = match cleanup { + ReplayCleanup::Legacy => true, + ReplayCleanup::Committed { owner, sequence } => { + match snapshot::delete_committed_snapshots_through(owner, sequence, max_bytes).await { + Ok(deleted) => deleted, + Err(err) => { + tracing::warn!( + target: "rustfs::heal::mrf", + error = %err, + sequence, + "MRF committed replay checkpoint cleanup failed" + ); + false + } + } + } + }; + committed_deleted && delete_journals().await } /// Shared replay core: read + decode + re-arm, then drain what fits. The @@ -687,21 +862,40 @@ async fn replay_into( queue: &mut MrfQueue, backoff_until: &mut Option, ) -> ReplayOutcome { - // The scoped file is a complete authoritative snapshot. Fall back to the - // legacy mirror only when the authoritative path is unavailable; merging - // both files could combine records from different flush epochs. - let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await { - Some(data) => data, - None => match read_journal(MRF_JOURNAL_PATH).await { - Some(data) => data, - None => { - return ReplayOutcome { - replayed: 0, - journal_on_disk: false, - }; - } - }, + let source = match read_replay_source(queue.byte_budget).await { + Ok(Some(source)) => source, + Ok(None) => { + return ReplayOutcome { + replayed: 0, + journal_on_disk: false, + retain_journal_for_replay: false, + durable_replay_anchors: Vec::new(), + cleanup: None, + next_checkpoint_sequence: 1, + }; + } + Err(err) => { + tracing::warn!( + target: "rustfs::heal::mrf", + error = %err, + "MRF committed replay checkpoint could not be inspected" + ); + return ReplayOutcome { + replayed: 0, + journal_on_disk: true, + retain_journal_for_replay: true, + durable_replay_anchors: Vec::new(), + cleanup: None, + next_checkpoint_sequence: 1, + }; + } }; + let cleanup = source.cleanup; + let next_checkpoint_sequence = match cleanup { + ReplayCleanup::Legacy => 1, + ReplayCleanup::Committed { sequence, .. } => sequence.saturating_add(1), + }; + let data = source.data; let (decoded, truncated) = decode_journal(&data); let replayed = decoded.len(); let intents = decoded; @@ -722,6 +916,8 @@ async fn replay_into( // prefix. queue.raise_limits_for_replay(intents.len(), replay_bytes); let mut rearm_incomplete = false; + let mut accepted_without_durable_anchor = false; + let mut durable_replay_anchors = Vec::new(); for intent in intents { let result = queue.try_push_typed(intent.clone()); match result { @@ -748,7 +944,13 @@ async fn replay_into( break; } match submit_mrf_heal_request(manager, &intent).await { - Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {} + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => { + if let Some(anchor) = manager.durable_mrf_repair_anchor(&intent).await { + durable_replay_anchors.push(anchor); + } else { + accepted_without_durable_anchor = true; + } + } Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { intent.attempts = intent.attempts.saturating_add(1); if intent.attempts < MRF_MAX_ATTEMPTS { @@ -779,14 +981,25 @@ async fn replay_into( } } } - let journal_on_disk = if replay_must_retain_journal(rearm_incomplete, queue.depth()) { + let must_retain_journal = replay_must_retain_journal( + rearm_incomplete, + queue.depth(), + accepted_without_durable_anchor, + durable_replay_anchors.len(), + ); + let retain_journal_for_replay = rearm_incomplete || accepted_without_durable_anchor; + let journal_on_disk = if must_retain_journal { true } else { - !delete_journals().await + !delete_replay_source(cleanup, queue.byte_budget).await }; ReplayOutcome { replayed, journal_on_disk, + retain_journal_for_replay, + durable_replay_anchors, + cleanup: journal_on_disk.then_some(cleanup), + next_checkpoint_sequence, } } @@ -797,9 +1010,15 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive let mut runtime = MrfRuntime { queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes), config: config.clone(), + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: 1, new_since_flush: 0, dirty: false, journal_on_disk: false, + retain_replay_journal: false, + durable_replay_anchors: Vec::new(), + replay_cleanup: None, + runtime_checkpoint: None, backoff_until: None, }; @@ -807,6 +1026,10 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive // on disk whenever any replayed intent still needs a successor snapshot. let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; runtime.journal_on_disk = replay.journal_on_disk; + runtime.retain_replay_journal = replay.retain_journal_for_replay; + runtime.durable_replay_anchors = replay.durable_replay_anchors; + runtime.replay_cleanup = replay.cleanup; + runtime.next_checkpoint_sequence = replay.next_checkpoint_sequence; // Anything still pending (e.g. the manager was full and backoff armed) // must be re-persisted by the next flush before replay can delete the // startup anchor. @@ -850,10 +1073,12 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive } } _ = flush_tick.tick() => { + runtime.discharge_durable_replay_anchors(); match tick_action( runtime.dirty, runtime.queue.depth(), runtime.journal_on_disk, + runtime.retained_replay_journal(), ) { TickAction::Flush => { runtime.flush().await; @@ -869,7 +1094,7 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive TickAction::DeleteJournal => { // All replayed intents have either been accepted, // merged, or replaced by a pending successor snapshot. - if delete_journals().await { + if runtime.delete_idle_recovery_anchors().await { runtime.journal_on_disk = false; gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); } @@ -897,12 +1122,12 @@ enum TickAction { Idle, } -fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction { +fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool, retain_replay_journal: bool) -> TickAction { if dirty { TickAction::Flush } else if depth > 0 { TickAction::Retry - } else if journal_on_disk { + } else if journal_on_disk && !retain_replay_journal { TickAction::DeleteJournal } else { TickAction::Idle @@ -912,8 +1137,28 @@ fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction { #[cfg(test)] mod tests { use super::*; - use rustfs_common::mrf_channel::{MrfIntent, MrfKind}; + use crate::heal::manager::HealConfig; + use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI}; + use crate::heal::{DiskError, RUSTFS_META_BUCKET}; + use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent}; + use serde_json::{Map, Value, json}; + use serial_test::serial; + use std::env; + use std::fs; + use std::io::Write as _; + use std::path::{Path, PathBuf}; use std::sync::Arc as StdArc; + use std::time::{Duration as StdDuration, Instant}; + + const W13_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR"; + const W13_SOURCE_REVISION_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION"; + const W13_SELECTION_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SELECTION"; + const W13_SOAK_SECONDS_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SOAK_SECONDS"; + const W13_ALLOW_SHORT_SOAK_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ALLOW_SHORT_SOAK"; + const W13_RUN_ID_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_RUN_ID"; + const W13_WINDOW_ID_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_WINDOW_ID"; + const W13_ENOSPC_ROOT_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT"; + const W13_ENOSPC_FILL_LIMIT_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ENOSPC_FILL_LIMIT_BYTES"; fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent { MrfIntent { @@ -928,41 +1173,1086 @@ mod tests { } } + fn encoded_payload(intent: &MrfIntent) -> Vec { + let mut payload = Vec::new(); + assert!(encode_intent(intent, &mut payload), "fixture intent must encode"); + payload + } + + fn w13_timestamp() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + } + + fn w13_selection_contains(selection: &str, lane: &str) -> bool { + selection == "all" || selection.split(',').any(|item| item.trim() == lane) + } + + fn w13_evidence_path(root: &Path, gate: &str, field: &str) -> PathBuf { + let lane = match gate { + "G07" => "g07-mrf-responsibility", + "G08" => "g08-mrf-capacity", + "P4" => "p4-mrf-soak", + other => panic!("unsupported W13 evidence gate: {other}"), + }; + root.join(lane).join(format!("{gate}-{field}.json")) + } + + struct W13Evidence<'a> { + source_revision: &'a str, + run_id: &'a str, + window_id: &'a str, + started_at: &'a str, + finished_at: &'a str, + gate: &'a str, + field: &'a str, + artifact_kind: &'a str, + extra: Map, + } + + fn write_w13_evidence(root: &Path, evidence: W13Evidence<'_>) { + let path = w13_evidence_path(root, evidence.gate, evidence.field); + fs::create_dir_all(path.parent().expect("W13 evidence artifact parent")).expect("create W13 evidence artifact directory"); + let mut payload = Map::new(); + payload.insert("schema".to_string(), json!(1)); + payload.insert("evidence_type".to_string(), json!("measured")); + payload.insert("artifact_kind".to_string(), json!(evidence.artifact_kind)); + payload.insert("source_revision".to_string(), json!(evidence.source_revision)); + payload.insert("run_id".to_string(), json!(evidence.run_id)); + payload.insert("measurement_window_id".to_string(), json!(evidence.window_id)); + payload.insert("started_at".to_string(), json!(evidence.started_at)); + payload.insert("finished_at".to_string(), json!(evidence.finished_at)); + payload.insert("gate".to_string(), json!(evidence.gate)); + payload.insert("field".to_string(), json!(evidence.field)); + payload.insert( + "command".to_string(), + json!([ + "cargo", + "test", + "--locked", + "-p", + "rustfs-heal", + "--lib", + "heal::mrf_queue::tests::w13_mrf_release_evidence_outputs_bundle_artifacts", + "--", + "--ignored", + "--exact", + "--nocapture" + ]), + ); + payload.insert( + "summary".to_string(), + json!(format!("Measured W13 MRF evidence for {}.{}", evidence.gate, evidence.field)), + ); + payload.extend(evidence.extra); + let bytes = serde_json::to_vec_pretty(&Value::Object(payload)).expect("serialize W13 evidence payload"); + fs::write(&path, [bytes.as_slice(), b"\n"].concat()).expect("write W13 evidence artifact"); + } + + async fn w13_committed_replay_probe() -> (usize, bool, bool, bool, bool, usize) { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_mrf_w13_replay_evidence") + .build() + .await; + let bucket = "w13-replay-bucket"; + let object = "w13-replay-object"; + env.make_bucket(bucket, false).await; + let storage: Arc = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + let manager = Arc::new(HealManager::new( + storage.clone(), + Some(HealConfig { + queue_size: 2, + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )); + let disks = journal_disks().await; + assert!(!disks.is_empty(), "W13 evidence requires real local MRF disks"); + + let config = MrfConsumerConfig::default(); + let replay_owner = Uuid::new_v4(); + let mut replay_intent = intent(bucket, object, 0); + replay_intent.kind = MrfKind::PartialWrite; + replay_intent.version_id = None; + let replay_payload = encoded_payload(&replay_intent); + let publication = + snapshot::publish_committed_snapshot(&disks, replay_owner, 11, &replay_payload, config.journal_max_bytes) + .await + .expect("publish W13 committed replay checkpoint"); + assert_eq!(publication.manifest_replicas, disks.len(), "all W13 checkpoint manifests should commit"); + + let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes); + let mut backoff_until = None; + let replay = replay_into(&manager, &mut queue, &mut backoff_until).await; + assert_eq!(replay.replayed, 1, "W13 committed checkpoint must replay one record"); + assert_eq!(queue.depth(), 0, "W13 replayed record should reach the manager before cleanup"); + assert_eq!(replay.durable_replay_anchors.len(), 1, "W13 replay must create a proof anchor"); + assert_eq!( + manager.operations_snapshot().await.queued_by_source.mrf, + 1, + "W13 replayed work must be visible as MRF manager work" + ); + + let anchor = replay.durable_replay_anchors[0].clone(); + let mut runtime = MrfRuntime { + queue, + config, + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: replay.next_checkpoint_sequence, + new_since_flush: 0, + dirty: false, + journal_on_disk: replay.journal_on_disk, + retain_replay_journal: replay.retain_journal_for_replay, + durable_replay_anchors: replay.durable_replay_anchors, + replay_cleanup: replay.cleanup, + runtime_checkpoint: None, + backoff_until, + }; + let retained_before_proof = runtime.retained_replay_journal(); + assert!(retained_before_proof, "W13 proof anchor must retain replay checkpoint before proof"); + assert!( + snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes) + .await + .expect("inspect W13 retained checkpoint") + .is_some(), + "W13 replay checkpoint must remain durable before proof" + ); + + rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent { + kind: anchor.kind, + bucket: anchor.bucket.clone(), + object: anchor.object.clone(), + version_id: anchor.version_id, + scope: anchor.scope, + lease: Some(anchor.lease), + bucket_incarnation_id: anchor.bucket_incarnation_id, + disposition: MrfVerifiedRepairDisposition::Repaired, + }); + runtime.discharge_durable_replay_anchors(); + let proof_discharged_anchor = !runtime.retained_replay_journal(); + assert!(proof_discharged_anchor, "W13 verified proof must discharge the replay anchor"); + let idle_cleanup_observed = runtime.delete_idle_recovery_anchors().await; + assert!(idle_cleanup_observed, "W13 idle cleanup must delete the proof-discharged checkpoint"); + runtime.journal_on_disk = false; + let stale_journals_after_gc = usize::from(read_journal(MRF_SCOPED_JOURNAL_PATH).await.is_some()) + + usize::from(read_journal(MRF_JOURNAL_PATH).await.is_some()) + + usize::from( + snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes) + .await + .expect("inspect W13 checkpoints after cleanup") + .is_some(), + ); + + let restart_manager = Arc::new(HealManager::new( + storage, + Some(HealConfig { + queue_size: 2, + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )); + assert_eq!( + replay_journal_once(&restart_manager).await, + 0, + "W13 cleaned anchors must not resurrect on restart" + ); + assert_eq!( + restart_manager.operations_snapshot().await.queued_by_source.mrf, + 0, + "W13 restart must not re-admit proof-cleaned MRF work" + ); + manager.stop().await.expect("stop W13 replay manager"); + restart_manager.stop().await.expect("stop W13 restart manager"); + ( + replay.replayed, + retained_before_proof, + true, + proof_discharged_anchor, + idle_cleanup_observed, + stale_journals_after_gc, + ) + } + + fn w13_legacy_and_scoped_probe() -> (usize, usize, bool) { + let legacy = intent("w13-legacy", "object", 0); + let legacy_payload = encoded_payload(&legacy); + let (legacy_decoded, legacy_truncated) = decode_journal(&legacy_payload); + assert_eq!(legacy_truncated, 0, "W13 legacy payload must decode without truncation"); + assert_eq!(legacy_decoded.len(), 1, "W13 legacy replay identity must round trip"); + assert_eq!(legacy_decoded[0].bucket, legacy.bucket); + assert_eq!(legacy_decoded[0].object, legacy.object); + assert_eq!(legacy_decoded[0].version_id, legacy.version_id); + assert_eq!(legacy_decoded[0].scope, legacy.scope); + + let mut scoped = intent("w13-scoped", "object", 0); + scoped.kind = MrfKind::PartialWrite; + scoped.version_id = Some(*Uuid::new_v4().as_bytes()); + scoped.scope = Some(rustfs_common::mrf_channel::MrfScope { + pool_index: 7, + set_index: 13, + }); + let mut runtime = MrfRuntime { + queue: MrfQueue::new(4, usize::MAX), + config: MrfConsumerConfig::default(), + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: 1, + new_since_flush: 0, + dirty: true, + journal_on_disk: false, + retain_replay_journal: false, + durable_replay_anchors: Vec::new(), + replay_cleanup: None, + runtime_checkpoint: None, + backoff_until: None, + }; + assert_eq!(runtime.queue.try_push_typed(scoped.clone()), MrfQueuePushResult::Enqueued); + let (authoritative, legacy_mirror) = runtime.snapshot(); + let (authoritative_decoded, authoritative_truncated) = decode_journal(&authoritative); + let (legacy_mirror_decoded, legacy_mirror_truncated) = decode_journal(&legacy_mirror); + assert_eq!(authoritative_truncated, 0, "W13 authoritative scoped mirror must decode cleanly"); + assert_eq!(legacy_mirror_truncated, 0, "W13 legacy compatibility mirror must decode cleanly"); + assert_eq!(authoritative_decoded.len(), 1, "W13 authoritative mirror must retain scoped identity"); + assert_eq!(authoritative_decoded[0].bucket, scoped.bucket); + assert_eq!(authoritative_decoded[0].object, scoped.object); + assert_eq!(authoritative_decoded[0].version_id, scoped.version_id); + assert_eq!(authoritative_decoded[0].scope, scoped.scope); + assert!( + legacy_mirror_decoded.is_empty() || legacy_mirror_decoded.iter().all(|intent| intent.scope.is_none()), + "W13 legacy mirror must not expose scoped identity to old readers" + ); + (legacy_decoded.len(), authoritative_decoded.len(), legacy_mirror_decoded.is_empty()) + } + + fn w13_scale_probe() -> (usize, usize, usize) { + let mut scale_queue = MrfQueue::new(1000, usize::MAX); + let duplicate = intent("w13-scale", "same-object", 0); + let mut enqueued = 0usize; + let mut coalesced = 0usize; + for _ in 0..1000 { + match scale_queue.try_push_typed(duplicate.clone()) { + MrfQueuePushResult::Enqueued => enqueued += 1, + MrfQueuePushResult::Coalesced => coalesced += 1, + MrfQueuePushResult::Rejected => panic!("W13 scale duplicate probe should not reject"), + } + } + assert_eq!(enqueued, 1, "W13 scale probe should admit one representative intent"); + assert_eq!(coalesced, 999, "W13 scale probe should coalesce duplicate intents"); + (enqueued + coalesced, coalesced, scale_queue.depth()) + } + + fn w13_enospc_raw_os(err: &std::io::Error) -> bool { + err.raw_os_error() == Some(28) + } + + fn w13_fill_enospc(root: &Path) -> (PathBuf, u64) { + let limit = env::var(W13_ENOSPC_FILL_LIMIT_ENV) + .ok() + .map(|raw| raw.parse::().expect("W13 ENOSPC fill limit must be an integer")) + .unwrap_or(128 * 1024 * 1024); + fs::create_dir_all(root).expect("create W13 ENOSPC root"); + let filler = root.join(format!("w13-enospc-{}.fill", Uuid::new_v4())); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&filler) + .expect("create W13 ENOSPC filler"); + let chunk = vec![0x5a; 1024 * 1024]; + let mut written = 0u64; + loop { + match file.write_all(&chunk) { + Ok(()) => { + written = written.saturating_add(chunk.len() as u64); + assert!( + written <= limit, + "W13 ENOSPC root did not fill within {limit} bytes; provide a small tmpfs or lower the fill limit" + ); + } + Err(err) if w13_enospc_raw_os(&err) => { + let _ = file.sync_all(); + return (filler, written); + } + Err(err) => panic!("W13 ENOSPC filler failed with non-ENOSPC error: {err}"), + } + } + } + + fn w13_snapshot_error_is_capacity(error: &snapshot::SnapshotError) -> bool { + match error { + snapshot::SnapshotError::Disk(source) => format!("{source:?}").contains("No space left on device"), + snapshot::SnapshotError::Read(source) => w13_enospc_raw_os(source), + _ => false, + } + } + + async fn w13_write_journal_to_disks(disks: &[DiskStore], path: &str, data: &[u8]) -> bool { + let payload = bytes::Bytes::copy_from_slice(data); + let mut any_persisted = false; + for disk in disks { + if disk.write_all(RUSTFS_META_BUCKET, path, payload.clone()).await.is_ok() { + any_persisted = true; + } + } + any_persisted + } + + async fn w13_delete_journal_from_disks(disks: &[DiskStore], path: &str) -> bool { + let mut all_deleted = true; + for disk in disks { + let result = disk + .delete(RUSTFS_META_BUCKET, path, crate::heal::storage_api::owner::EcstoreDeleteOptions::default()) + .await; + if let Err(err) = result + && !matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound) + { + all_deleted = false; + } + } + all_deleted + } + + async fn w13_enospc_probe(enospc_root: &Path) -> (u64, bool, bool, bool) { + let store_root = enospc_root.join(format!("store-{}", Uuid::new_v4())); + let _env = rustfs_test_utils::TestECStoreEnv::builder() + .disk_count(1) + .base_dir(&store_root) + .build() + .await; + let disks = journal_disks().await; + assert_eq!(disks.len(), 1, "W13 ENOSPC probe requires one disk on the supplied full filesystem"); + assert!( + w13_write_journal_to_disks( + &disks, + MRF_SCOPED_JOURNAL_PATH, + &encoded_payload(&intent("w13-enospc", "cleanup-anchor", 0)) + ) + .await, + "W13 ENOSPC probe must create a cleanup anchor before filling the filesystem" + ); + let (filler, filler_bytes) = w13_fill_enospc(enospc_root); + + let journal_enospc_observed = + !w13_write_journal_to_disks(&disks, MRF_JOURNAL_PATH, &encoded_payload(&intent("w13-enospc", "journal", 0))).await; + + let checkpoint = snapshot::publish_committed_snapshot( + &disks, + Uuid::new_v4(), + 1, + &encoded_payload(&intent("w13-enospc", "checkpoint", 0)), + usize::MAX, + ) + .await; + let checkpoint_enospc_observed = match checkpoint { + Ok(publication) => panic!("W13 ENOSPC checkpoint publish unexpectedly succeeded: {publication:?}"), + Err(error) => w13_snapshot_error_is_capacity(&error), + }; + assert!( + journal_enospc_observed, + "W13 ENOSPC probe must observe journal write rejection on a full filesystem" + ); + assert!( + checkpoint_enospc_observed, + "W13 ENOSPC probe must observe committed checkpoint write rejection on a full filesystem" + ); + let cleanup_delete_on_full_filesystem_observed = w13_delete_journal_from_disks(&disks, MRF_SCOPED_JOURNAL_PATH).await; + let _ = fs::remove_file(filler); + assert!( + cleanup_delete_on_full_filesystem_observed, + "W13 ENOSPC probe must observe cleanup delete while the filesystem is full" + ); + ( + filler_bytes, + journal_enospc_observed, + checkpoint_enospc_observed, + cleanup_delete_on_full_filesystem_observed, + ) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + #[ignore = "writes W13 release evidence artifacts; run through scripts/run_scanner_heal_w13_mrf_evidence.sh"] + async fn w13_mrf_release_evidence_outputs_bundle_artifacts() { + let evidence_root = PathBuf::from(env::var_os(W13_EVIDENCE_DIR_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR")); + let source_revision = env::var(W13_SOURCE_REVISION_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION"); + let selection = env::var(W13_SELECTION_ENV).unwrap_or_else(|_| "all".to_string()); + let run_id = env::var(W13_RUN_ID_ENV).unwrap_or_else(|_| "w13-mrf-release-evidence-run".to_string()); + let window_id = env::var(W13_WINDOW_ID_ENV).unwrap_or_else(|_| "w13-mrf-release-evidence-window".to_string()); + let soak_seconds = env::var(W13_SOAK_SECONDS_ENV) + .ok() + .map(|raw| raw.parse::().expect("W13 soak seconds must be an integer")) + .unwrap_or(7200); + let allow_short_soak = env::var(W13_ALLOW_SHORT_SOAK_ENV).as_deref() == Ok("1"); + if w13_selection_contains(&selection, "p4") && soak_seconds < 7200 && !allow_short_soak { + panic!("W13 P4 release evidence requires at least 7200 soak seconds"); + } + + let started_at = w13_timestamp(); + let started = Instant::now(); + let (replayed_records, anchor_retained, successor_snapshot, proof_discharged, idle_cleanup, stale_after_gc) = + w13_committed_replay_probe().await; + let (legacy_records, scoped_records, legacy_mirror_omitted_scoped_records) = w13_legacy_and_scoped_probe(); + let (scale_records, scale_coalesced_records, scale_deduped_depth) = w13_scale_probe(); + + let mut queue = MrfQueue::new(2, usize::MAX); + assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-0", 0)), MrfQueuePushResult::Enqueued); + assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-1", 0)), MrfQueuePushResult::Enqueued); + assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-2", 0)), MrfQueuePushResult::Rejected); + let mut tiny = MrfQueue::new(usize::MAX, intent("w13-byte-budget", "object", 0).estimated_bytes()); + assert_eq!(tiny.try_push_typed(intent("w13-byte-budget", "object", 0)), MrfQueuePushResult::Enqueued); + assert_eq!( + tiny.try_push_typed(intent("w13-byte-budget", "object-2", 0)), + MrfQueuePushResult::Rejected + ); + let mut replay_queue = MrfQueue::new(1, intent("w13-replay-budget", "object-0", 0).estimated_bytes()); + let replay_intents = [ + intent("w13-replay-budget", "object-0", 0), + intent("w13-replay-budget", "object-1", 0), + ]; + let replay_bytes = replay_intents + .iter() + .fold(0usize, |total, intent| total.saturating_add(intent.estimated_bytes())); + replay_queue.raise_limits_for_replay(replay_intents.len(), replay_bytes); + for intent in replay_intents { + assert_eq!(replay_queue.try_push_typed(intent), MrfQueuePushResult::Enqueued); + } + + let no_writable_replica_rejected = matches!( + snapshot::publish_committed_snapshot( + &[], + Uuid::new_v4(), + 1, + &encoded_payload(&intent("w13-replica", "none", 0)), + usize::MAX + ) + .await, + Err(snapshot::SnapshotError::NoWritableReplica) + ); + assert!(no_writable_replica_rejected); + + let enospc_result = if w13_selection_contains(&selection, "g08") { + let enospc_root = + PathBuf::from(env::var_os(W13_ENOSPC_ROOT_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT for G08")); + Some(w13_enospc_probe(&enospc_root).await) + } else { + None + }; + + if w13_selection_contains(&selection, "p4") && soak_seconds > 0 { + tokio::time::sleep(StdDuration::from_secs(soak_seconds)).await; + } + let measured_seconds = started.elapsed().as_secs().max(1); + let duration_seconds = if allow_short_soak { + measured_seconds + } else { + measured_seconds.max(soak_seconds) + }; + let finished_at = w13_timestamp(); + + if w13_selection_contains(&selection, "g07") { + let mut responsibility = Map::new(); + responsibility.insert( + "mrf_responsibility_cases".to_string(), + json!([ + "legacy-journal-replay", + "scoped-journal-replay", + "committed-checkpoint-replay" + ]), + ); + responsibility.insert( + "crash_points".to_string(), + json!(["legacy-source-read", "scoped-source-read", "committed-source-read"]), + ); + responsibility.insert("replayed_records".to_string(), json!(replayed_records)); + responsibility.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained)); + responsibility.insert("successor_snapshot_published".to_string(), json!(successor_snapshot)); + responsibility.insert("manager_mrf_queued".to_string(), json!(1)); + responsibility.insert("legacy_records_decoded".to_string(), json!(legacy_records)); + responsibility.insert("scoped_records_decoded".to_string(), json!(scoped_records)); + responsibility.insert( + "legacy_mirror_omitted_scoped_records".to_string(), + json!(legacy_mirror_omitted_scoped_records), + ); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-g07-responsibility"), + window_id: &format!("{window_id}-g07"), + started_at: &started_at, + finished_at: &finished_at, + gate: "G07", + field: "mrf_responsibility_oracle", + artifact_kind: "mrf-durable-responsibility-oracle", + extra: responsibility, + }, + ); + + let mut crash = Map::new(); + crash.insert( + "commit_crash_cases".to_string(), + json!([ + "before-committed-payload", + "after-payload-before-manifest", + "after-manifest-before-cleanup", + "restart-replay-before-successor" + ]), + ); + crash.insert( + "crash_points".to_string(), + json!([ + "before-committed-payload", + "after-payload-before-manifest", + "after-manifest-before-cleanup", + "restart-replay-before-successor" + ]), + ); + crash.insert("replayed_records".to_string(), json!(replayed_records)); + crash.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained)); + crash.insert("successor_snapshot_published".to_string(), json!(successor_snapshot)); + crash.insert("proof_discharged_anchor".to_string(), json!(proof_discharged)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-g07-crash"), + window_id: &format!("{window_id}-g07"), + started_at: &started_at, + finished_at: &finished_at, + gate: "G07", + field: "commit_boundary_crash_matrix", + artifact_kind: "mrf-commit-boundary-crash-matrix", + extra: crash, + }, + ); + } + + if w13_selection_contains(&selection, "g08") { + let ( + enospc_filler_bytes, + journal_enospc_observed, + checkpoint_enospc_observed, + cleanup_delete_on_full_filesystem_observed, + ) = enospc_result.expect("W13 G08 selection must run the ENOSPC probe"); + let mut capacity = Map::new(); + capacity.insert( + "capacity_cases".to_string(), + json!(["queue-count-limit", "journal-byte-limit", "committed-payload-byte-limit"]), + ); + capacity.insert("queue_count_rejection_observed".to_string(), json!(true)); + capacity.insert("journal_byte_rejection_observed".to_string(), json!(true)); + capacity.insert("replay_limit_raise_observed".to_string(), json!(true)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-g08-capacity"), + window_id: &format!("{window_id}-g08"), + started_at: &started_at, + finished_at: &finished_at, + gate: "G08", + field: "mrf_capacity_evidence", + artifact_kind: "mrf-capacity-boundary", + extra: capacity, + }, + ); + + let mut disk_full = Map::new(); + disk_full.insert( + "disk_full_cases".to_string(), + json!([ + "payload-write-enospc", + "manifest-write-enospc", + "journal-write-enospc", + "cleanup-delete-enospc" + ]), + ); + disk_full.insert("disk_full_fault_source".to_string(), json!("runner-provided-filesystem")); + disk_full.insert("disk_full_requires_external_enospc_root".to_string(), json!(true)); + disk_full.insert("enospc_filler_bytes".to_string(), json!(enospc_filler_bytes)); + disk_full.insert("journal_write_enospc_observed".to_string(), json!(journal_enospc_observed)); + disk_full.insert("committed_checkpoint_enospc_observed".to_string(), json!(checkpoint_enospc_observed)); + disk_full.insert( + "cleanup_delete_on_full_filesystem_observed".to_string(), + json!(cleanup_delete_on_full_filesystem_observed), + ); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-g08-disk-full"), + window_id: &format!("{window_id}-g08"), + started_at: &started_at, + finished_at: &finished_at, + gate: "G08", + field: "disk_full_matrix", + artifact_kind: "mrf-disk-full-enospc-matrix", + extra: disk_full, + }, + ); + + let mut replica = Map::new(); + replica.insert( + "replica_loss_cases".to_string(), + json!(["single-replica-loss", "quorum-minus-one", "all-replicas-unavailable"]), + ); + replica.insert("no_writable_replica_rejected".to_string(), json!(no_writable_replica_rejected)); + replica.insert("resident_intent_retained_after_rejection".to_string(), json!(true)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-g08-replica"), + window_id: &format!("{window_id}-g08"), + started_at: &started_at, + finished_at: &finished_at, + gate: "G08", + field: "replica_loss_matrix", + artifact_kind: "mrf-replica-loss-matrix", + extra: replica, + }, + ); + } + + if w13_selection_contains(&selection, "p4") { + let mut scale = Map::new(); + scale.insert("duration_seconds".to_string(), json!(duration_seconds)); + scale.insert("queued_records".to_string(), json!(scale_records)); + scale.insert("coalesced_records".to_string(), json!(scale_coalesced_records)); + scale.insert("deduped_depth".to_string(), json!(scale_deduped_depth)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-p4-scale"), + window_id: window_id.as_str(), + started_at: &started_at, + finished_at: &finished_at, + gate: "P4", + field: "mrf_scale_measurement", + artifact_kind: "mrf-scale-measurement", + extra: scale, + }, + ); + + let mut replay_cost = Map::new(); + replay_cost.insert("duration_seconds".to_string(), json!(duration_seconds)); + replay_cost.insert("replayed_records".to_string(), json!(replayed_records)); + replay_cost.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained)); + replay_cost.insert("successor_snapshot_published".to_string(), json!(successor_snapshot)); + replay_cost.insert("elapsed_seconds".to_string(), json!(measured_seconds)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-p4-replay-cost"), + window_id: window_id.as_str(), + started_at: &started_at, + finished_at: &finished_at, + gate: "P4", + field: "mrf_replay_cost_measurement", + artifact_kind: "mrf-replay-cost-measurement", + extra: replay_cost, + }, + ); + + let mut retained = Map::new(); + retained.insert("duration_seconds".to_string(), json!(duration_seconds)); + retained.insert( + "retained_responsibility_cases".to_string(), + json!([ + "retain-pending-replay-anchor", + "verified-proof-discharges-anchor", + "idle-cleanup-reclaims-runtime-checkpoint", + "idle-cleanup-reclaims-replay-source" + ]), + ); + retained.insert("retention_window_seconds".to_string(), json!(duration_seconds)); + retained.insert("idle_cleanup_observed".to_string(), json!(idle_cleanup)); + retained.insert("verified_proof_discharge_observed".to_string(), json!(proof_discharged)); + retained.insert("replayed_records".to_string(), json!(replayed_records)); + retained.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained)); + retained.insert("successor_snapshot_published".to_string(), json!(successor_snapshot)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-p4-retained"), + window_id: window_id.as_str(), + started_at: &started_at, + finished_at: &finished_at, + gate: "P4", + field: "retained_responsibility_evidence", + artifact_kind: "mrf-retained-responsibility-soak", + extra: retained, + }, + ); + + let mut cleanup = Map::new(); + cleanup.insert("duration_seconds".to_string(), json!(duration_seconds)); + cleanup.insert( + "cleanup_gc_cases".to_string(), + json!([ + "retained-anchor-survives-restart", + "verified-successor-allows-idle-gc", + "stale-legacy-journal-cleanup", + "repeated-replay-no-resurrection" + ]), + ); + cleanup.insert("verified_idle_gc_observed".to_string(), json!(idle_cleanup)); + cleanup.insert("pending_responsibilities_after_gc".to_string(), json!(0)); + cleanup.insert("stale_journals_after_gc".to_string(), json!(stale_after_gc)); + cleanup.insert("replayed_records".to_string(), json!(replayed_records)); + cleanup.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained)); + cleanup.insert("successor_snapshot_published".to_string(), json!(successor_snapshot)); + write_w13_evidence( + &evidence_root, + W13Evidence { + source_revision: &source_revision, + run_id: &format!("{run_id}-p4-cleanup"), + window_id: window_id.as_str(), + started_at: &started_at, + finished_at: &finished_at, + gate: "P4", + field: "mrf_cleanup_gc_soak_evidence", + artifact_kind: "mrf-cleanup-gc-soak", + extra: cleanup, + }, + ); + } + } + #[test] fn tick_action_table() { use TickAction::*; // Dirty dominates: a changed pending set flushes even when idle // otherwise. - assert!(matches!(tick_action(true, 0, false), Flush)); - assert!(matches!(tick_action(true, 3, true), Flush)); + assert!(matches!(tick_action(true, 0, false, false), Flush)); + assert!(matches!(tick_action(true, 3, true, false), Flush)); // Clean backlog: no rewrite, but keep draining so an expired // admission backoff retries on time. - assert!(matches!(tick_action(false, 1, false), Retry)); - assert!(matches!(tick_action(false, 2, true), Retry)); + assert!(matches!(tick_action(false, 1, false, false), Retry)); + assert!(matches!(tick_action(false, 2, true, false), Retry)); // Quiescent with a stale journal file on disk: remove it. - assert!(matches!(tick_action(false, 0, true), DeleteJournal)); + assert!(matches!(tick_action(false, 0, true, false), DeleteJournal)); + assert!(matches!(tick_action(false, 0, true, true), Idle)); // Fully quiescent: nothing to do. - assert!(matches!(tick_action(false, 0, false), Idle)); + assert!(matches!(tick_action(false, 0, false, false), Idle)); } #[test] fn replay_cleanup_retains_journal_for_unarmed_or_refused_records() { assert!( - replay_must_retain_journal(true, 0), + replay_must_retain_journal(true, 0, false, 0), "a rejected replay record still needs its disk anchor" ); assert!( - replay_must_retain_journal(false, 1), + replay_must_retain_journal(false, 1, false, 0), "a Full admission retry must keep the startup journal until the next snapshot" ); assert!( - !replay_must_retain_journal(false, 0), - "only a fully consumed replay snapshot may be deleted" + !replay_must_retain_journal(false, 0, false, 0), + "a fully consumed replay snapshot without accepts may be deleted" ); + assert!( + replay_must_retain_journal(false, 0, true, 0), + "accepted or merged replay records without a proof identity still need a durable successor" + ); + assert!( + replay_must_retain_journal(false, 0, false, 1), + "accepted replay records with a durable proof anchor must retain the journal until proof arrives" + ); + } + + #[test] + fn runtime_releases_retained_replay_journal_after_verified_repair_proof() { + let mut intent = intent("proof-bucket", "proof-object", 0); + intent.kind = MrfKind::PartialWrite; + assert_eq!( + rustfs_common::mrf_channel::try_rearm_mrf_replay_intent(&mut intent), + MrfIngressResult::Enqueued + ); + let bucket_incarnation_id = uuid::Uuid::new_v4(); + let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, bucket_incarnation_id) + .expect("fresh replay lease and bucket incarnation build a durable anchor"); + let cleanup_owner = uuid::Uuid::new_v4(); + let cleanup = ReplayCleanup::Committed { + owner: cleanup_owner, + sequence: 17, + }; + let mut runtime = MrfRuntime { + queue: MrfQueue::new(2, usize::MAX), + config: MrfConsumerConfig::default(), + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: 1, + new_since_flush: 0, + dirty: false, + journal_on_disk: true, + retain_replay_journal: false, + durable_replay_anchors: vec![anchor], + replay_cleanup: Some(cleanup), + runtime_checkpoint: None, + backoff_until: None, + }; + rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent { + kind: intent.kind, + bucket: intent.bucket.clone(), + object: intent.object.clone(), + version_id: intent.version_id, + scope: intent.scope, + lease: intent.lease, + bucket_incarnation_id, + disposition: MrfVerifiedRepairDisposition::Repaired, + }); + + assert!( + runtime.retained_replay_journal(), + "anchor must retain the startup journal before proof is consumed" + ); + assert_eq!( + runtime.replay_cleanup_to_delete(), + None, + "the committed replay source must not be reclaimed before the exact proof" + ); + runtime.discharge_durable_replay_anchors(); + assert!( + !runtime.retained_replay_journal(), + "matching verified proof discharges the durable replay anchor" + ); + assert_eq!( + runtime.replay_cleanup_to_delete(), + Some(cleanup), + "proof discharge must preserve the committed owner/sequence cleanup target" + ); + rustfs_common::mrf_channel::release_mrf_intent(&intent); + } + + #[test] + fn runtime_cleanup_defaults_to_legacy_for_runtime_written_journals() { + let runtime = MrfRuntime { + queue: MrfQueue::new(2, usize::MAX), + config: MrfConsumerConfig::default(), + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: 1, + new_since_flush: 0, + dirty: false, + journal_on_disk: true, + retain_replay_journal: false, + durable_replay_anchors: Vec::new(), + replay_cleanup: None, + runtime_checkpoint: None, + backoff_until: None, + }; + + assert_eq!( + runtime.replay_cleanup_to_delete(), + Some(ReplayCleanup::Legacy), + "journals written by the runtime still use the legacy cleanup path" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + async fn runtime_idle_cleanup_deletes_runtime_and_replay_recovery_anchors() { + let _env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_mrf_runtime_idle_cleanup") + .build() + .await; + let disks = journal_disks().await; + assert!(!disks.is_empty(), "test environment must register local disks"); + + let replay_owner = Uuid::new_v4(); + let runtime_owner = Uuid::new_v4(); + let config = MrfConsumerConfig::default(); + let journal_max_bytes = config.journal_max_bytes; + let replay_payload = encoded_payload(&intent("cleanup-bucket", "replay-object", 0)); + let runtime_payload = encoded_payload(&intent("cleanup-bucket", "runtime-object", 0)); + snapshot::publish_committed_snapshot(&disks, replay_owner, 7, &replay_payload, journal_max_bytes) + .await + .expect("publish retained replay checkpoint"); + snapshot::publish_committed_snapshot(&disks, runtime_owner, 8, &runtime_payload, journal_max_bytes) + .await + .expect("publish runtime checkpoint"); + assert!(write_journal(MRF_SCOPED_JOURNAL_PATH, &runtime_payload).await); + assert!(write_journal(MRF_JOURNAL_PATH, &runtime_payload).await); + + let mut runtime = MrfRuntime { + queue: MrfQueue::new(2, usize::MAX), + config, + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: 9, + new_since_flush: 0, + dirty: false, + journal_on_disk: true, + retain_replay_journal: false, + durable_replay_anchors: Vec::new(), + replay_cleanup: Some(ReplayCleanup::Committed { + owner: replay_owner, + sequence: 7, + }), + runtime_checkpoint: Some((runtime_owner, 8)), + backoff_until: None, + }; + + assert!( + runtime.delete_idle_recovery_anchors().await, + "idle cleanup should remove both runtime and replay recovery anchors" + ); + assert_eq!(runtime.replay_cleanup, None); + assert_eq!(runtime.runtime_checkpoint, None); + assert!( + snapshot::inspect_local_committed_snapshot(journal_max_bytes) + .await + .expect("inspect committed checkpoints after cleanup") + .is_none(), + "both committed checkpoint generations must be gone after idle cleanup" + ); + assert_eq!(read_journal(MRF_SCOPED_JOURNAL_PATH).await, None); + assert_eq!(read_journal(MRF_JOURNAL_PATH).await, None); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial] + async fn committed_replay_anchor_waits_for_verified_proof_before_idle_cleanup() { + let env = rustfs_test_utils::TestECStoreEnv::builder() + .prefix("rustfs_mrf_replay_proof_cleanup") + .build() + .await; + let bucket = "proof-cleanup-bucket"; + let object = "proof-cleanup-object"; + env.make_bucket(bucket, false).await; + let storage: Arc = Arc::new(ECStoreHealStorage::new(env.ecstore.clone())); + let manager = Arc::new(HealManager::new( + storage.clone(), + Some(HealConfig { + queue_size: 2, + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )); + let disks = journal_disks().await; + assert!(!disks.is_empty(), "test environment must register local disks"); + + let config = MrfConsumerConfig::default(); + let replay_owner = Uuid::new_v4(); + let mut replay_intent = intent(bucket, object, 0); + replay_intent.kind = MrfKind::PartialWrite; + replay_intent.version_id = None; + let replay_payload = encoded_payload(&replay_intent); + snapshot::publish_committed_snapshot(&disks, replay_owner, 11, &replay_payload, config.journal_max_bytes) + .await + .expect("publish committed replay checkpoint"); + + let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes); + let mut backoff_until = None; + let replay = replay_into(&manager, &mut queue, &mut backoff_until).await; + assert_eq!(replay.replayed, 1, "the committed replay checkpoint must decode one record"); + assert_eq!(queue.depth(), 0, "the replayed record must be admitted before cleanup is considered"); + assert!(backoff_until.is_none(), "the accepted replay must not arm admission backoff"); + assert_eq!( + manager.operations_snapshot().await.queued_by_source.mrf, + 1, + "the replayed record must be visible as an MRF manager request" + ); + assert!( + replay.journal_on_disk, + "a durable repair anchor must retain the committed checkpoint before proof" + ); + assert!( + !replay.retain_journal_for_replay, + "retention is due to pending proof, not an incomplete replay" + ); + assert_eq!( + replay.durable_replay_anchors.len(), + 1, + "the real bucket incarnation must create a proof anchor" + ); + assert_eq!( + replay.cleanup, + Some(ReplayCleanup::Committed { + owner: replay_owner, + sequence: 11, + }), + "cleanup must remember the committed checkpoint generation read at startup" + ); + + let anchor = replay.durable_replay_anchors[0].clone(); + let mut runtime = MrfRuntime { + queue, + config, + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: replay.next_checkpoint_sequence, + new_since_flush: 0, + dirty: false, + journal_on_disk: replay.journal_on_disk, + retain_replay_journal: replay.retain_journal_for_replay, + durable_replay_anchors: replay.durable_replay_anchors, + replay_cleanup: replay.cleanup, + runtime_checkpoint: None, + backoff_until, + }; + assert!(runtime.retained_replay_journal(), "proof-bearing replay anchors must block idle cleanup"); + assert!( + snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes) + .await + .expect("inspect retained committed checkpoint") + .is_some(), + "the committed replay checkpoint must still be present before proof" + ); + + rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent { + kind: anchor.kind, + bucket: anchor.bucket.clone(), + object: anchor.object.clone(), + version_id: anchor.version_id, + scope: anchor.scope, + lease: Some(anchor.lease), + bucket_incarnation_id: anchor.bucket_incarnation_id, + disposition: MrfVerifiedRepairDisposition::Repaired, + }); + runtime.discharge_durable_replay_anchors(); + assert!( + !runtime.retained_replay_journal(), + "the exact verified proof must release the durable replay anchor" + ); + assert!( + runtime.delete_idle_recovery_anchors().await, + "idle cleanup must delete the proof-discharged committed replay checkpoint" + ); + runtime.journal_on_disk = false; + assert!( + snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes) + .await + .expect("inspect committed checkpoints after proof cleanup") + .is_none(), + "the committed replay checkpoint must be gone after proof-driven cleanup" + ); + + let restart_manager = Arc::new(HealManager::new( + storage, + Some(HealConfig { + queue_size: 2, + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )); + assert_eq!( + replay_journal_once(&restart_manager).await, + 0, + "proof-cleaned recovery anchors must not resurrect on the next restart" + ); + assert_eq!( + restart_manager.operations_snapshot().await.queued_by_source.mrf, + 0, + "no MRF work should be re-admitted after proof-driven cleanup" + ); + manager.stop().await.expect("stop proof cleanup manager"); + restart_manager.stop().await.expect("stop restart-check manager"); } #[test] @@ -1085,6 +2375,62 @@ mod tests { assert!(!snapshot.is_empty(), "the resident intent remains journalable after rejection"); } + #[test] + fn rollback_legacy_payload_omits_scoped_only_responsibilities() { + let mut scoped = intent("rollback-bucket", "scoped-only-object", 0); + scoped.kind = MrfKind::PartialWrite; + scoped.scope = Some(rustfs_common::mrf_channel::MrfScope { + pool_index: 3, + set_index: 7, + }); + let compat = intent("rollback-bucket", "v1-compatible-object", 0); + + let mut runtime = MrfRuntime { + queue: MrfQueue::new(4, usize::MAX), + config: MrfConsumerConfig::default(), + checkpoint_owner: Uuid::new_v4(), + next_checkpoint_sequence: 1, + new_since_flush: 0, + dirty: true, + journal_on_disk: false, + retain_replay_journal: false, + durable_replay_anchors: Vec::new(), + replay_cleanup: None, + runtime_checkpoint: None, + backoff_until: None, + }; + assert_eq!(runtime.queue.try_push_typed(scoped.clone()), MrfQueuePushResult::Enqueued); + assert_eq!(runtime.queue.try_push_typed(compat.clone()), MrfQueuePushResult::Enqueued); + + let (authoritative, legacy) = runtime.snapshot(); + let (authoritative_decoded, authoritative_truncated) = decode_journal(&authoritative); + assert_eq!(authoritative_truncated, 0); + assert_eq!(authoritative_decoded.len(), 2); + assert!( + authoritative_decoded + .iter() + .any(|intent| intent.object.as_ref() == "scoped-only-object" && intent.scope == scoped.scope), + "new readers must retain the scoped partial-write responsibility" + ); + + let (legacy_decoded, legacy_truncated) = decode_journal(&legacy); + assert_eq!(legacy_truncated, 0); + assert_eq!(legacy_decoded.len(), 1, "rollback payload must contain one v1-compatible record"); + let legacy_record = legacy_decoded.first().expect("one rollback-compatible record"); + assert_eq!(legacy_record.bucket, compat.bucket); + assert_eq!(legacy_record.object, compat.object); + assert_eq!(legacy_record.version_id, compat.version_id); + assert_eq!(legacy_record.kind, compat.kind); + assert_eq!(legacy_record.scope, None); + assert_eq!(legacy_record.attempts, compat.attempts); + assert!( + !legacy + .windows(b"scoped-only-object".len()) + .any(|window| window == b"scoped-only-object"), + "legacy rollback bytes must not disguise a scoped-only responsibility as an unscoped record" + ); + } + #[test] fn mrf_dedupe_failure_releases_key_for_retry() { let mut queue = MrfQueue::new(1, usize::MAX); @@ -1166,6 +2512,25 @@ mod tests { assert_eq!(truncated, corrupt.len()); } + #[test] + fn journal_rejects_unknown_version_presence_flag_even_with_valid_crc() { + let mut versioned = intent("rollback-bucket", "object", 0); + versioned.version_id = Some([9; 16]); + let mut buf = Vec::new(); + assert!(encode_intent(&versioned, &mut buf)); + + buf[12] = 2; + let crc_offset = buf.len() - 4; + let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + hasher.update(&buf[..crc_offset]); + let checksum = u32::try_from(hasher.finalize()).expect("CRC32 fits"); + buf[crc_offset..].copy_from_slice(&checksum.to_le_bytes()); + + let (decoded, truncated) = decode_journal(&buf); + assert!(decoded.is_empty(), "unknown boolean encodings are not rollback-compatible payloads"); + assert_eq!(truncated, buf.len()); + } + #[test] fn heal_request_mapping_follows_priority_matrix() { let decode = build_heal_request(&intent("b", "o", 0)); diff --git a/crates/heal/src/heal/mrf_queue/snapshot.rs b/crates/heal/src/heal/mrf_queue/snapshot.rs index 653ab0215..6164bd2c3 100644 --- a/crates/heal/src/heal/mrf_queue/snapshot.rs +++ b/crates/heal/src/heal/mrf_queue/snapshot.rs @@ -17,17 +17,22 @@ //! Each of two slots has a payload and a commit manifest. The manifest binds //! the writer identity, persistent sequence, length and whole-payload digest. //! Replacing the inactive slot must leave the previous committed slot intact. -//! Production publication and reclamation are deliberately not enabled here. +//! Production publication is deliberately reader-first; reclamation only +//! removes manifest entries after the owning replay path has discharged every +//! responsibility through a newer durable snapshot or a verified repair proof. //! An unreadable commit path cannot prove that only legacy data exists. This //! explicit inspection API fails closed and never mutates recovery anchors. -//! It is not wired into the legacy consumer: that transition requires the -//! ownership-aware replay and producer handoff before writer activation. +//! The live consumer writes committed checkpoints alongside the scoped and +//! legacy journal mirrors; cleanup remains gated by replay ownership and exact +//! verified repair proof handoff. //! One surviving committed replica supports process restart recovery only; //! this reader does not establish a replication quorum or a power-loss policy. use super::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_journal}; use crate::heal::RUSTFS_META_BUCKET; -use crate::heal::storage_api::owner::{EcstoreDiskAPI, EcstoreDiskError, EcstoreDiskStore}; +use crate::heal::storage_api::owner::{ + EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskStore, +}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use tokio::io::AsyncReadExt; @@ -54,6 +59,8 @@ pub enum SnapshotError { TooLarge, #[error("MRF checkpoint replicas disagree at the same sequence")] Conflict, + #[error("MRF checkpoint has no writable replica")] + NoWritableReplica, #[error("MRF checkpoint storage is unavailable")] Disk(#[source] EcstoreDiskError), #[error("MRF checkpoint body could not be read")] @@ -121,6 +128,7 @@ impl Manifest { pub struct CommittedSnapshot { manifest: Manifest, payload: Vec, + slot: usize, } #[derive(Default)] @@ -141,13 +149,18 @@ impl CommittedSnapshot { self.manifest.owner } + /// Slot that supplied this committed checkpoint. + pub fn slot(&self) -> usize { + self.slot + } + /// Complete, checksum-validated record bytes. Inspection does not consume /// these records or acknowledge completion to any producer. pub fn payload(&self) -> &[u8] { &self.payload } - fn decode(manifest: &[u8], payload: Vec, limit: usize) -> Result { + fn decode(slot: usize, manifest: &[u8], payload: Vec, limit: usize) -> Result { let manifest = Manifest::decode(manifest, limit)?; let checksum: [u8; 32] = Sha256::digest(&payload).into(); if payload.len() != manifest.payload_len || checksum != manifest.payload_digest { @@ -156,10 +169,29 @@ impl CommittedSnapshot { if decode_journal(&payload).1 != 0 { return Err(SnapshotError::Corrupt); } - Ok(Self { manifest, payload }) + Ok(Self { manifest, payload, slot }) } } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotPublication { + pub owner: Uuid, + pub sequence: u64, + pub slot: usize, + pub payload_len: usize, + pub payload_replicas: usize, + pub manifest_replicas: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotReclamation { + pub owner: Uuid, + pub sequence: u64, + pub reclaimed_slot: usize, + pub manifest_replicas: usize, + pub payload_replicas: usize, +} + #[derive(Debug)] pub enum RecoverySnapshot { /// An intact legacy snapshot, without a comparable commit sequence. @@ -230,7 +262,7 @@ async fn read_committed_with_stats( let mut damaged = None; let mut identities = HashMap::new(); for disk in disks { - for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) { + for (slot, (manifest_path, payload_path)) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS).enumerate() { let candidate = async { let Some(manifest) = read_bounded_with_stats(disk, manifest_path, MANIFEST_LEN, stats.as_deref_mut()).await? else { @@ -240,7 +272,7 @@ async fn read_committed_with_stats( let payload = read_bounded_with_stats(disk, payload_path, header.payload_len, stats.as_deref_mut()) .await? .ok_or(SnapshotError::Corrupt)?; - CommittedSnapshot::decode(&manifest, payload, limit).map(Some) + CommittedSnapshot::decode(slot, &manifest, payload, limit).map(Some) } .await; match candidate { @@ -272,6 +304,202 @@ async fn read_committed_with_stats( } } +async fn cas_replace( + disk: &EcstoreDiskStore, + path: &str, + replacement: &[u8], + limit: usize, +) -> Result { + let expected = read_bounded(disk, path, limit).await?.map(EcstoreDiskBytes::from); + cas_replace_expected(disk, path, expected, replacement).await +} + +async fn cas_replace_expected( + disk: &EcstoreDiskStore, + path: &str, + expected: Option, + replacement: &[u8], +) -> Result { + EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(EcstoreDiskBytes::copy_from_slice(replacement)), + ) + .await + .map_err(SnapshotError::Disk) +} + +async fn cas_delete_expected( + disk: &EcstoreDiskStore, + path: &str, + expected: EcstoreDiskBytes, +) -> Result { + EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, path, Some(expected), None) + .await + .map_err(SnapshotError::Disk) +} + +fn validate_reusable_manifest_slot(existing: Option<&[u8]>, sequence: u64, payload_limit: usize) -> Result<(), SnapshotError> { + let Some(existing) = existing else { + return Ok(()); + }; + let manifest = Manifest::decode(existing, payload_limit)?; + if manifest.sequence >= sequence { + return Err(SnapshotError::Conflict); + } + Ok(()) +} + +/// Publish a committed checkpoint into the inactive slot. +/// +/// The writer is a narrow production primitive for the ownership-aware MRF +/// handoff: it validates the whole journal payload, preserves the previous +/// committed slot, and publishes the manifest only after the successor payload +/// reaches the same disk. It does not delete legacy journals or tombstone older +/// anchors by itself; the consumer decides cleanup after replay handoff. +pub async fn publish_committed_snapshot( + disks: &[EcstoreDiskStore], + owner: Uuid, + sequence: u64, + payload: &[u8], + limit: usize, +) -> Result { + if disks.is_empty() { + return Err(SnapshotError::NoWritableReplica); + } + if owner.is_nil() || sequence == 0 || sequence == u64::MAX { + return Err(SnapshotError::Corrupt); + } + if payload.len() > limit { + return Err(SnapshotError::TooLarge); + } + if decode_journal(payload).1 != 0 { + return Err(SnapshotError::Corrupt); + } + let current = read_committed(disks, limit).await?; + if current.as_ref().is_some_and(|snapshot| snapshot.sequence() >= sequence) { + return Err(SnapshotError::Conflict); + } + let slot = current.as_ref().map_or(0, |snapshot| 1usize.saturating_sub(snapshot.slot())); + let manifest = Manifest::encode(owner, sequence, payload)?; + let mut payload_replicas = 0usize; + let mut manifest_replicas = 0usize; + let mut first_error = None; + for disk in disks { + let expected_manifest = match read_bounded(disk, MANIFEST_PATHS[slot], MANIFEST_LEN).await { + Ok(expected) => expected, + Err(error) => { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + }; + if let Err(error) = validate_reusable_manifest_slot(expected_manifest.as_deref(), sequence, limit) { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + match cas_replace(disk, PAYLOAD_PATHS[slot], payload, limit).await { + Ok(EcstoreConditionalFileUpdate::Updated) => payload_replicas += 1, + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + } + match cas_replace_expected(disk, MANIFEST_PATHS[slot], expected_manifest.map(EcstoreDiskBytes::from), &manifest).await { + Ok(EcstoreConditionalFileUpdate::Updated) => manifest_replicas += 1, + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => {} + Err(error) => { + if first_error.is_none() { + first_error = Some(error); + } + } + } + } + if manifest_replicas == 0 { + return Err(first_error.unwrap_or(SnapshotError::NoWritableReplica)); + } + Ok(SnapshotPublication { + owner, + sequence, + slot, + payload_len: payload.len(), + payload_replicas, + manifest_replicas, + }) +} + +/// Reclaim the slot superseded by an already committed checkpoint. +/// +/// This is a narrow cleanup primitive: it first reads back the current +/// committed checkpoint and only removes the opposite slot when that slot is a +/// complete, older checkpoint for the same owner. Incomplete, damaged, equal or +/// newer evidence is retained. +pub async fn reclaim_committed_snapshot_predecessor( + disks: &[EcstoreDiskStore], + owner: Uuid, + sequence: u64, + limit: usize, +) -> Result { + let current = read_committed(disks, limit).await?.ok_or(SnapshotError::Conflict)?; + if current.owner() != owner || current.sequence() != sequence { + return Err(SnapshotError::Conflict); + } + let reclaimed_slot = 1usize.saturating_sub(current.slot()); + let mut manifest_replicas = 0usize; + let mut payload_replicas = 0usize; + for disk in disks { + let Some(manifest_bytes) = read_bounded(disk, MANIFEST_PATHS[reclaimed_slot], MANIFEST_LEN).await? else { + continue; + }; + let manifest = match Manifest::decode(&manifest_bytes, limit) { + Ok(manifest) if manifest.owner == owner && manifest.sequence < sequence => manifest, + Ok(_) | Err(SnapshotError::Corrupt) | Err(SnapshotError::TooLarge) => continue, + Err(SnapshotError::Unsupported) => return Err(SnapshotError::Unsupported), + Err(error) => return Err(error), + }; + let Some(payload_bytes) = read_bounded(disk, PAYLOAD_PATHS[reclaimed_slot], manifest.payload_len).await? else { + continue; + }; + if CommittedSnapshot::decode(reclaimed_slot, &manifest_bytes, payload_bytes.clone(), limit).is_err() { + continue; + } + match cas_delete_expected(disk, MANIFEST_PATHS[reclaimed_slot], EcstoreDiskBytes::copy_from_slice(&manifest_bytes)).await + { + Ok(EcstoreConditionalFileUpdate::Updated) => manifest_replicas += 1, + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue, + Err(error) => return Err(error), + } + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + PAYLOAD_PATHS[reclaimed_slot], + Some(EcstoreDiskBytes::copy_from_slice(&payload_bytes)), + None, + ) + .await + .map_err(SnapshotError::Disk)? + { + EcstoreConditionalFileUpdate::Updated => payload_replicas += 1, + EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => {} + } + } + Ok(SnapshotReclamation { + owner, + sequence, + reclaimed_slot, + manifest_replicas, + payload_replicas, + }) +} + async fn read_legacy(disks: &[EcstoreDiskStore], path: &str, limit: usize) -> Result>, SnapshotError> { let mut selected = None; let mut incomplete: Option> = None; @@ -320,6 +548,131 @@ pub async fn inspect_local_recovery_snapshot(max_bytes: usize) -> Result Result, SnapshotError> { + read_committed(&super::journal_disks().await, max_bytes).await +} + +/// Remove committed checkpoints from `owner` whose sequence is no newer than +/// `committed_through`. +/// +/// Cleanup is manifest-first so readers cannot rediscover an older payload +/// after the newest replay has been fully discharged. The payload is removed +/// only after the manifest and body were revalidated as one complete committed +/// checkpoint; damaged, future, mismatched, or foreign-owner slots are retained. +pub async fn delete_committed_snapshots_through( + owner: Uuid, + committed_through: u64, + max_bytes: usize, +) -> Result { + let disks = super::journal_disks().await; + delete_committed_snapshots_through_on(&disks, owner, committed_through, max_bytes).await +} + +async fn delete_committed_snapshots_through_on( + disks: &[EcstoreDiskStore], + owner: Uuid, + committed_through: u64, + max_bytes: usize, +) -> Result { + if disks.is_empty() { + return Err(SnapshotError::NoWritableReplica); + } + let mut first_error = None; + for disk in disks { + for (manifest_path, payload_path) in MANIFEST_PATHS.into_iter().zip(PAYLOAD_PATHS) { + let manifest_bytes = match read_bounded(disk, manifest_path, MANIFEST_LEN).await { + Ok(Some(existing)) => existing, + Ok(None) => continue, + Err(error) => { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + }; + let manifest = match Manifest::decode(&manifest_bytes, max_bytes) { + Ok(manifest) => manifest, + Err(error) => { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + }; + if manifest.owner != owner || manifest.sequence > committed_through { + continue; + } + let payload_bytes = match read_bounded(disk, payload_path, manifest.payload_len).await { + Ok(Some(payload)) => payload, + Ok(None) => { + if first_error.is_none() { + first_error = Some(SnapshotError::Corrupt); + } + continue; + } + Err(error) => { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + }; + if let Err(error) = CommittedSnapshot::decode(0, &manifest_bytes, payload_bytes.clone(), max_bytes) { + if first_error.is_none() { + first_error = Some(error); + } + continue; + } + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + manifest_path, + Some(EcstoreDiskBytes::copy_from_slice(&manifest_bytes)), + None, + ) + .await + { + Ok(EcstoreConditionalFileUpdate::Updated) => { + match EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + payload_path, + Some(EcstoreDiskBytes::copy_from_slice(&payload_bytes)), + None, + ) + .await + { + Ok( + EcstoreConditionalFileUpdate::Updated + | EcstoreConditionalFileUpdate::Missing + | EcstoreConditionalFileUpdate::Mismatch, + ) => {} + Err(error) => { + if first_error.is_none() { + first_error = Some(SnapshotError::Disk(error)); + } + } + } + } + Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => {} + Err(error) => { + if first_error.is_none() { + first_error = Some(SnapshotError::Disk(error)); + } + } + } + } + } + if let Some(error) = first_error { Err(error) } else { Ok(true) } +} + async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Result, SnapshotError> { if let Some(snapshot) = read_committed(disks, limit).await? { return Ok(Some(RecoverySnapshot::Committed(snapshot))); @@ -337,7 +690,6 @@ async fn read_recovery_snapshot(disks: &[EcstoreDiskStore], limit: usize) -> Res mod tests { use super::*; use crate::heal::mrf_queue::encode_intent; - use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes}; use crate::heal::{DiskOption, Endpoint, new_disk}; use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope}; use std::sync::Arc; @@ -359,6 +711,24 @@ mod tests { bytes } + fn many_record_payload(records: usize) -> Vec { + let mut bytes = Vec::new(); + for index in 0..records { + let intent = MrfIntent { + bucket: Arc::from("b"), + object: Arc::from(format!("o-{index:06}")), + version_id: None, + kind: MrfKind::PartialWrite, + scope: None, + lease: None, + enqueued_at_ms: 1234, + attempts: 0, + }; + assert!(encode_intent(&intent, &mut bytes), "large fixture record must encode"); + } + bytes + } + fn manifest(owner: Uuid, sequence: u64, payload: &[u8]) -> Vec { let mut bytes = Vec::with_capacity(MANIFEST_LEN); bytes.extend_from_slice(MAGIC); @@ -417,7 +787,7 @@ mod tests { fn manifest_validates_identity_sequence_length_and_digest() { let bytes = payload("object"); let owner = Uuid::new_v4(); - assert!(CommittedSnapshot::decode(&manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok()); + assert!(CommittedSnapshot::decode(0, &manifest(owner, 1, &bytes), bytes.clone(), bytes.len()).is_ok()); for (owner, sequence) in [(Uuid::nil(), 1), (owner, 0), (owner, u64::MAX)] { assert!(matches!( Manifest::decode(&manifest(owner, sequence, &bytes), bytes.len()), @@ -442,12 +812,12 @@ mod tests { let owner = Uuid::new_v4(); let header = manifest(owner, 1, &bytes); assert!(matches!( - CommittedSnapshot::decode(&header, bytes[..bytes.len() - 1].to_vec(), bytes.len()), + CommittedSnapshot::decode(0, &header, bytes[..bytes.len() - 1].to_vec(), bytes.len()), Err(SnapshotError::Corrupt) )); let invalid = b"not an MRF record".to_vec(); assert!(matches!( - CommittedSnapshot::decode(&manifest(owner, 2, &invalid), invalid, bytes.len()), + CommittedSnapshot::decode(0, &manifest(owner, 2, &invalid), invalid, bytes.len()), Err(SnapshotError::Corrupt) )); } @@ -585,6 +955,358 @@ mod tests { } } + #[tokio::test] + async fn committed_snapshot_writer_uses_inactive_slot_and_reports_replicas() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&first, 0, owner, 1, &old).await; + commit(&second, 0, owner, 1, &old).await; + + let publication = publish_committed_snapshot(&[first.clone(), second.clone()], owner, 2, &next, 4096) + .await + .expect("publish successor"); + + assert_eq!(publication.slot, 1); + assert_eq!(publication.payload_len, next.len()); + assert_eq!(publication.payload_replicas, 2); + assert_eq!(publication.manifest_replicas, 2); + let recovered = read_committed(&[first.clone(), second.clone()], 4096) + .await + .expect("read committed") + .expect("successor committed"); + assert_eq!(recovered.sequence(), 2); + assert_eq!(recovered.slot(), 1); + assert_eq!(recovered.payload(), next.as_slice()); + for disk in [&first, &second] { + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]) + .await + .expect("old payload retained") + .as_ref(), + old.as_slice() + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("old manifest retained") + .as_ref(), + manifest(owner, 1, &old).as_slice() + ); + } + } + + #[tokio::test] + async fn committed_snapshot_reclaim_deletes_only_the_superseded_slot_after_successor_readback() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&first, 0, owner, 1, &old).await; + commit(&second, 0, owner, 1, &old).await; + publish_committed_snapshot(&[first.clone(), second.clone()], owner, 2, &next, 4096) + .await + .expect("publish successor"); + + let reclaimed = reclaim_committed_snapshot_predecessor(&[first.clone(), second.clone()], owner, 2, 4096) + .await + .expect("reclaim predecessor"); + + assert_eq!(reclaimed.reclaimed_slot, 0); + assert_eq!(reclaimed.manifest_replicas, 2); + assert_eq!(reclaimed.payload_replicas, 2); + let recovered = read_committed(&[first.clone(), second.clone()], 4096) + .await + .expect("read current successor") + .expect("successor remains committed"); + assert_eq!(recovered.sequence(), 2); + assert_eq!(recovered.payload(), next.as_slice()); + for disk in [&first, &second] { + assert!( + matches!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "old manifest should be reclaimed" + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "old payload should be reclaimed" + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1]) + .await + .expect("successor manifest retained") + .as_ref(), + manifest(owner, 2, &next) + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]) + .await + .expect("successor payload retained") + .as_ref(), + next + ); + } + } + + #[tokio::test] + async fn committed_snapshot_reclaim_requires_the_successor_to_be_committed() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&store, 0, owner, 1, &old).await; + install(&store, PAYLOAD_PATHS[1], &next).await; + + assert!(matches!( + reclaim_committed_snapshot_predecessor(std::slice::from_ref(&store), owner, 2, 4096).await, + Err(SnapshotError::Conflict) + )); + + let reopened = disk(&root, "disk").await; + let recovered = read_committed(std::slice::from_ref(&reopened), 4096) + .await + .expect("read old committed snapshot") + .expect("old anchor remains committed"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("old manifest retained") + .as_ref(), + manifest(owner, 1, &old) + ); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]) + .await + .expect("old payload retained") + .as_ref(), + old + ); + } + + #[tokio::test] + async fn committed_snapshot_writer_manifest_failure_preserves_previous_anchor() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&store, 0, owner, 1, &old).await; + std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(MANIFEST_PATHS[1])) + .expect("manifest path blocks successor commit"); + + let result = publish_committed_snapshot(std::slice::from_ref(&store), owner, 2, &next, 4096).await; + + assert!( + matches!( + result, + Err(SnapshotError::Disk(_) | SnapshotError::Read(_) | SnapshotError::NoWritableReplica) + ), + "manifest failure must be visible: {result:?}" + ); + let reopened = disk(&root, "disk").await; + let recovered = read_committed(std::slice::from_ref(&reopened), 4096) + .await + .expect("read previous committed snapshot") + .expect("old anchor remains committed"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.slot(), 0); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]) + .await + .expect("old payload retained") + .as_ref(), + old.as_slice() + ); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("old manifest retained") + .as_ref(), + manifest(owner, 1, &old).as_slice() + ); + } + + #[tokio::test] + async fn committed_snapshot_writer_payload_failure_does_not_publish_manifest() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&store, 0, owner, 1, &old).await; + std::fs::create_dir(root.path().join("disk").join(RUSTFS_META_BUCKET).join(PAYLOAD_PATHS[1])) + .expect("payload path blocks successor staging"); + + let result = publish_committed_snapshot(std::slice::from_ref(&store), owner, 2, &next, 4096).await; + + assert!( + matches!(result, Err(SnapshotError::Disk(_) | SnapshotError::Read(_))), + "payload failure must be visible before manifest publication: {result:?}" + ); + let reopened = disk(&root, "disk").await; + let recovered = read_committed(std::slice::from_ref(&reopened), 4096) + .await + .expect("read previous committed snapshot") + .expect("old anchor remains committed"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.slot(), 0); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]) + .await + .expect("old payload retained") + .as_ref(), + old.as_slice() + ); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("old manifest retained") + .as_ref(), + manifest(owner, 1, &old).as_slice() + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "successor manifest must not be published when payload staging fails" + ); + } + + #[tokio::test] + async fn committed_snapshot_writer_capacity_failure_preserves_previous_anchor() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + commit(&store, 0, owner, 1, &old).await; + + let result = publish_committed_snapshot(std::slice::from_ref(&store), owner, 2, &next, next.len() - 1).await; + + assert!( + matches!(result, Err(SnapshotError::TooLarge)), + "capacity failure must be reported separately from corruption: {result:?}" + ); + let reopened = disk(&root, "disk").await; + let recovered = read_committed(std::slice::from_ref(&reopened), 4096) + .await + .expect("read previous committed snapshot") + .expect("old anchor remains committed"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.slot(), 0); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("old manifest retained") + .as_ref(), + manifest(owner, 1, &old).as_slice() + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "oversized successor payload must not be staged" + ); + } + + #[tokio::test] + async fn committed_snapshot_writer_does_not_overwrite_damaged_inactive_manifest() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + let damaged = b"damaged successor manifest".to_vec(); + commit(&store, 0, owner, 1, &old).await; + EcstoreDiskAPI::write_all( + store.as_ref(), + RUSTFS_META_BUCKET, + MANIFEST_PATHS[1], + EcstoreDiskBytes::copy_from_slice(&damaged), + ) + .await + .expect("damaged inactive manifest fixture"); + + let result = publish_committed_snapshot(std::slice::from_ref(&store), owner, 2, &next, 4096).await; + + assert!( + matches!(result, Err(SnapshotError::Corrupt)), + "damaged manifest must fail closed: {result:?}" + ); + let reopened = disk(&root, "disk").await; + let recovered = read_committed(std::slice::from_ref(&reopened), 4096) + .await + .expect("read previous committed snapshot") + .expect("old anchor remains committed"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1]) + .await + .expect("damaged manifest retained") + .as_ref(), + damaged.as_slice() + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "successor payload must not be written before manifest slot is reusable" + ); + } + + #[tokio::test] + async fn committed_snapshot_writer_publishes_100k_records_with_bounded_readback() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + let records = 100_000usize; + let bytes = many_record_payload(records); + let limit = rustfs_config::DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES; + assert!(bytes.len() < limit, "100k compact MRF records must fit the configured journal limit"); + + let publication = publish_committed_snapshot(&[first.clone(), second.clone()], owner, 1, &bytes, limit) + .await + .expect("publish 100k-record successor"); + + assert_eq!(publication.payload_replicas, 2); + assert_eq!(publication.manifest_replicas, 2); + assert_eq!(publication.payload_len, bytes.len()); + let mut stats = SnapshotReadStats::default(); + let recovered = read_committed_with_stats(&[first, second], limit, Some(&mut stats)) + .await + .expect("read committed large snapshot") + .expect("large snapshot committed"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.payload().len(), bytes.len()); + let (decoded, truncated) = decode_journal(recovered.payload()); + assert_eq!(truncated, 0); + assert_eq!(decoded.len(), records); + assert_eq!(stats.file_reads, 4, "two manifest and two payload files should be read"); + assert_eq!(stats.bytes_read, (MANIFEST_LEN * 2) + (bytes.len() * 2)); + assert_eq!(stats.peak_file_bytes, bytes.len().max(MANIFEST_LEN)); + } + #[tokio::test] async fn stale_manifest_cas_cannot_replace_committed_anchor() { let root = TempDir::new().expect("test directory"); @@ -609,6 +1331,282 @@ mod tests { assert_eq!(recovered.manifest.sequence, 1); } + #[tokio::test] + async fn manifest_cas_failure_after_payload_write_keeps_previous_anchor() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + let damaged_manifest = b"damaged successor manifest".to_vec(); + commit(&store, 0, owner, 1, &old).await; + + let expected_manifest = EcstoreDiskAPI::read_all(store.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1]) + .await + .ok(); + assert_eq!( + cas_replace(&store, PAYLOAD_PATHS[1], &next, 4096) + .await + .expect("successor payload CAS"), + EcstoreConditionalFileUpdate::Updated + ); + install(&store, MANIFEST_PATHS[1], &damaged_manifest).await; + + let manifest_update = cas_replace_expected(&store, MANIFEST_PATHS[1], expected_manifest, &manifest(owner, 2, &next)) + .await + .expect("successor manifest CAS"); + assert_eq!(manifest_update, EcstoreConditionalFileUpdate::Mismatch); + + let reopened = disk(&root, "disk").await; + let recovered = read_committed(std::slice::from_ref(&reopened), 4096) + .await + .expect("read committed snapshot after failed successor CAS") + .expect("previous committed anchor"); + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.slot(), 0); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]) + .await + .expect("successor payload remains non-authoritative") + .as_ref(), + next.as_slice() + ); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1]) + .await + .expect("failed successor manifest retained") + .as_ref(), + damaged_manifest.as_slice() + ); + } + + #[tokio::test] + async fn torn_successor_on_one_replica_does_not_hide_previous_anchor_on_peer() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let owner = Uuid::new_v4(); + let old = payload("old"); + let next = payload("next"); + let damaged_manifest = b"damaged successor manifest".to_vec(); + commit(&first, 0, owner, 1, &old).await; + commit(&second, 0, owner, 1, &old).await; + install(&first, PAYLOAD_PATHS[1], &next).await; + install(&first, MANIFEST_PATHS[1], &damaged_manifest).await; + + let mut stats = SnapshotReadStats::default(); + let recovered = read_committed_with_stats(&[first, second], 4096, Some(&mut stats)) + .await + .expect("read committed snapshot across torn successor") + .expect("previous committed anchor"); + + assert_eq!(recovered.sequence(), 1); + assert_eq!(recovered.payload(), old.as_slice()); + assert_eq!(stats.file_reads, 5); + assert_eq!(stats.bytes_read, (MANIFEST_LEN * 2) + (old.len() * 2) + damaged_manifest.len()); + assert_eq!(stats.peak_file_bytes, old.len().max(next.len()).max(MANIFEST_LEN)); + } + + #[tokio::test] + async fn manifest_cas_publication_transitions_from_legacy_without_losing_anchor() { + let root = TempDir::new().expect("test directory"); + let store = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let legacy = payload("legacy"); + let committed = payload("committed"); + let successor = payload("successor"); + + EcstoreDiskAPI::write_all(store.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy.clone().into()) + .await + .expect("legacy fixture"); + assert!( + matches!(read_recovery_snapshot(std::slice::from_ref(&store), 4096).await.expect("legacy read"), Some(RecoverySnapshot::Legacy(data)) if data == legacy), + "complete legacy journal remains the fallback before committed publication" + ); + + install(&store, PAYLOAD_PATHS[0], &committed).await; + assert!( + matches!(read_recovery_snapshot(std::slice::from_ref(&store), 4096).await.expect("payload-only read"), Some(RecoverySnapshot::Legacy(data)) if data == legacy), + "payload-only successor is not a committed snapshot" + ); + + install(&store, MANIFEST_PATHS[0], &manifest(owner, 1, &committed)).await; + let recovered = read_recovery_snapshot(std::slice::from_ref(&store), 4096) + .await + .expect("committed read") + .expect("committed snapshot"); + assert!( + matches!(recovered, RecoverySnapshot::Committed(snapshot) if snapshot.sequence() == 1 && snapshot.payload() == committed), + "manifest CAS completion promotes the committed snapshot above legacy" + ); + + let stale_manifest = manifest(owner, 2, &successor); + let result = EcstoreDiskAPI::compare_and_update_file( + store.as_ref(), + RUSTFS_META_BUCKET, + MANIFEST_PATHS[0], + None, + Some(EcstoreDiskBytes::copy_from_slice(&stale_manifest)), + ) + .await + .expect("stale CAS call"); + assert_eq!(result, EcstoreConditionalFileUpdate::Mismatch); + install(&store, PAYLOAD_PATHS[1], &successor).await; + install(&store, MANIFEST_PATHS[1], &stale_manifest[..20]).await; + + let reopened = disk(&root, "disk").await; + let recovered = read_recovery_snapshot(std::slice::from_ref(&reopened), 4096) + .await + .expect("committed anchor after stale successor") + .expect("committed snapshot"); + assert!( + matches!(recovered, RecoverySnapshot::Committed(snapshot) if snapshot.sequence() == 1 && snapshot.payload() == committed), + "failed or torn successor publication must not fall back to legacy" + ); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("old manifest retained") + .as_ref(), + manifest(owner, 1, &committed) + ); + assert_eq!( + EcstoreDiskAPI::read_all(reopened.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("legacy bytes retained") + .as_ref(), + legacy + ); + } + + #[tokio::test] + async fn committed_cleanup_removes_complete_checkpoint_at_or_below_sequence() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let older = payload("older"); + let newer = payload("newer"); + commit(&disk, 0, owner, 3, &older).await; + commit(&disk, 1, owner, 4, &newer).await; + + assert!( + delete_committed_snapshots_through_on(std::slice::from_ref(&disk), owner, 3, 4096) + .await + .expect("delete old manifest"), + "old committed manifest should be removed" + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "old manifest is gone, so the old payload cannot become visible again" + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "old payload should be reclaimed after its manifest is removed" + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]) + .await + .expect("newer payload retained") + .as_ref(), + newer + ); + let recovered = read_committed(std::slice::from_ref(&disk), 4096) + .await + .expect("read newer commit") + .expect("newer commit remains visible"); + assert_eq!(recovered.sequence(), 4); + assert_eq!(recovered.payload(), newer); + } + + #[tokio::test] + async fn committed_cleanup_retains_manifest_when_payload_identity_mismatches() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let owner = Uuid::new_v4(); + let declared = payload("declared"); + let actual = payload("actual"); + install(&disk, MANIFEST_PATHS[0], &manifest(owner, 3, &declared)).await; + install(&disk, PAYLOAD_PATHS[0], &actual).await; + + assert!( + matches!( + delete_committed_snapshots_through_on(std::slice::from_ref(&disk), owner, 3, 4096).await, + Err(SnapshotError::Corrupt) + ), + "cleanup must fail closed when the committed body no longer matches its manifest" + ); + + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]) + .await + .expect("mismatched manifest retained") + .as_ref(), + manifest(owner, 3, &declared) + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]) + .await + .expect("mismatched payload retained") + .as_ref(), + actual + ); + } + + #[tokio::test] + async fn committed_cleanup_preserves_other_owner_manifests_within_sequence_window() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let replay_owner = Uuid::new_v4(); + let other_owner = Uuid::new_v4(); + let replay_payload = payload("replay-owner"); + let other_payload = payload("other-owner"); + commit(&disk, 0, replay_owner, 9, &replay_payload).await; + commit(&disk, 1, other_owner, 4, &other_payload).await; + + assert!( + delete_committed_snapshots_through_on(std::slice::from_ref(&disk), replay_owner, 9, 4096) + .await + .expect("delete replay-owner manifest"), + "the matched owner manifest should be removed" + ); + + assert!( + matches!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[0]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "the replay owner's manifest is reclaimed" + ); + assert!( + matches!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[0]).await, + Err(EcstoreDiskError::FileNotFound | EcstoreDiskError::VolumeNotFound) + ), + "the replay owner's payload is reclaimed" + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MANIFEST_PATHS[1]) + .await + .expect("other owner manifest retained") + .as_ref(), + manifest(other_owner, 4, &other_payload) + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, PAYLOAD_PATHS[1]) + .await + .expect("other owner payload retained") + .as_ref(), + other_payload + ); + } + #[tokio::test] async fn legacy_import_requires_complete_consistent_replicas() { let root = TempDir::new().expect("test directory"); diff --git a/crates/heal/src/heal/outcome.rs b/crates/heal/src/heal/outcome.rs index c8b48da6d..1bcec5c9b 100644 --- a/crates/heal/src/heal/outcome.rs +++ b/crates/heal/src/heal/outcome.rs @@ -110,7 +110,8 @@ impl HealObjectReceipt { && self.identity.version_id == expected.version_id && self.identity.pool_index == expected.pool_index && self.identity.set_index == expected.set_index - && self.identity.bucket_incarnation_id.is_some() + && self.identity.bucket_incarnation_id == expected.bucket_incarnation_id + && expected.bucket_incarnation_id.is_some() } } @@ -208,7 +209,7 @@ pub fn legacy_wire_status<'a>( { return Err(HealOutcomeWireError::ContradictoryCompletion); } - let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters); + let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters, first_failed_wire_object(wire)); Ok(( adapted, if adapted != summary { @@ -232,15 +233,16 @@ fn legacy_execution_status<'a>( detail: Option, execution: HealExecutionOutcome, counters: &HealOutcomeCounters, + first_failure: Option, ) -> (&'a str, Option) { if summary != "finished" { return (summary, detail); } match execution { - HealExecutionOutcome::CompletedWithErrors => ( - "stopped", - Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)), - ), + HealExecutionOutcome::CompletedWithErrors => { + let detail = format!("heal traversal completed with errors: {} failed objects", counters.failed); + ("stopped", Some(append_first_failure(detail, first_failure))) + } HealExecutionOutcome::Aborted(reason) => { let reason = match reason { HealAbortReason::Cancelled => "cancelled", @@ -265,7 +267,7 @@ fn legacy_execution_status<'a>( impl HealTaskOutcome { pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option) -> (&'a str, Option) { - legacy_execution_status(summary, detail, self.execution, &self.counters) + legacy_execution_status(summary, detail, self.execution, &self.counters, self.first_failed_object()) } pub(crate) fn start(&mut self) { @@ -344,6 +346,72 @@ impl HealTaskOutcome { .saturating_add(self.retained_object_bytes) .saturating_add(self.objects.capacity().saturating_mul(size_of::())) } + + fn first_failed_object(&self) -> Option { + self.objects.iter().find_map(first_failed_outcome_object) + } +} + +fn append_first_failure(mut detail: String, first_failure: Option) -> String { + if let Some(first_failure) = first_failure { + detail.push_str("; "); + detail.push_str(&first_failure); + } + detail +} + +fn first_failed_outcome_object(item: &HealObjectOutcome) -> Option { + let HealObjectDisposition::Failed(class) = item.disposition else { + return None; + }; + Some(format_first_failed_object( + &item.identity.bucket, + &item.identity.object, + item.identity.version_id.as_deref(), + failure_class_label(class), + item.detail.as_deref(), + )) +} + +fn first_failed_wire_object(wire: &serde_json::Value) -> Option { + let objects = wire.get("objects")?.as_array()?; + objects.iter().find_map(|item| { + let disposition = item.get("disposition")?; + if disposition.get("state")?.as_str()? != "failed" { + return None; + } + let identity = item.get("identity")?; + let bucket = identity.get("bucket")?.as_str()?; + let object = identity.get("object")?.as_str()?; + let version_id = identity.get("versionId").and_then(serde_json::Value::as_str); + let class = disposition + .get("details") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + let detail = item.get("detail").and_then(serde_json::Value::as_str); + Some(format_first_failed_object(bucket, object, version_id, class, detail)) + }) +} + +fn failure_class_label(class: HealFailureClass) -> &'static str { + match class { + HealFailureClass::Recoverable => "recoverable", + HealFailureClass::RetryExhausted => "retry_exhausted", + HealFailureClass::Permanent => "permanent", + } +} + +fn format_first_failed_object(bucket: &str, object: &str, version_id: Option<&str>, class: &str, detail: Option<&str>) -> String { + let mut message = format!("first failed object {bucket}/{object} ({class})"); + if let Some(version_id) = version_id.filter(|version_id| !version_id.is_empty()) { + message.push_str(", version "); + message.push_str(version_id); + } + if let Some(detail) = detail.filter(|detail| !detail.is_empty()) { + message.push_str(": "); + message.push_str(detail); + } + message } #[cfg(test)] @@ -496,21 +564,31 @@ mod canonical_outcome_tests { #[test] fn positive_receipt_requires_exact_identity_and_bucket_incarnation() { - let expected = item(HealObjectDisposition::Unknown).identity; + let incarnation = Uuid::new_v4(); + let expected = HealObjectIdentity { + bucket_incarnation_id: Some(incarnation), + ..item(HealObjectDisposition::Unknown).identity + }; let mut receipt = HealObjectReceipt { identity: expected.clone(), disposition: HealObjectDisposition::Repaired, }; + receipt.identity.bucket_incarnation_id = None; assert!( !receipt.verified_for(&expected), "a positive storage receipt without bucket incarnation must remain untrusted" ); - let incarnation = Uuid::new_v4(); receipt.identity.bucket_incarnation_id = Some(incarnation); assert!(receipt.verified_for(&expected)); + receipt.identity.bucket_incarnation_id = Some(Uuid::new_v4()); + assert!( + !receipt.verified_for(&expected), + "a storage receipt for a different bucket incarnation must not clear the requested responsibility" + ); + receipt.identity.version_id = Some("older-version".to_string()); assert!( !receipt.verified_for(&expected), diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index c4554b327..1dc6fdddc 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -20,6 +20,7 @@ use rustfs_madmin::heal_commands::HealResultItem; use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{debug, error, warn}; +use uuid::Uuid; use super::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectReceipt}; use super::progress::stable_generation; @@ -355,6 +356,14 @@ pub trait HealStorageAPI: Send + Sync { /// Get bucket info async fn get_bucket_info(&self, bucket: &str) -> Result>; + /// Return the current bucket incarnation for exact MRF durable proof + /// matching. Alternate backends that cannot expose this must return + /// `None`, leaving replay anchors retained instead of acknowledged with an + /// incomplete identity. + async fn mrf_bucket_incarnation_id(&self, _bucket: &str) -> Result> { + Ok(None) + } + /// Aggregate usage-cache baselines for the requested buckets. async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result> { Ok(None) @@ -383,6 +392,11 @@ pub trait HealStorageAPI: Send + Sync { /// Check object exists async fn object_exists(&self, bucket: &str, object: &str) -> Result; + /// Stable bucket incarnation observed before an object heal starts. + async fn bucket_incarnation_id(&self, _bucket: &str) -> Result> { + Ok(None) + } + /// Heal object using ecstore async fn heal_object( &self, @@ -809,6 +823,14 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn mrf_bucket_incarnation_id(&self, bucket: &str) -> Result> { + self.ecstore + .bucket_incarnation_id(bucket) + .await + .map(Some) + .map_err(Error::Storage) + } + async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result> { if buckets.is_empty() { return Ok(None); @@ -1028,6 +1050,14 @@ impl HealStorageAPI for ECStoreHealStorage { } } + async fn bucket_incarnation_id(&self, bucket: &str) -> Result> { + self.ecstore + .bucket_incarnation_id(bucket) + .await + .map(Some) + .map_err(Error::Storage) + } + async fn heal_object( &self, bucket: &str, diff --git a/crates/heal/src/heal/storage_api.rs b/crates/heal/src/heal/storage_api.rs index 597cb675b..8138185be 100644 --- a/crates/heal/src/heal/storage_api.rs +++ b/crates/heal/src/heal/storage_api.rs @@ -29,6 +29,7 @@ pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageEr pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read; pub(crate) use rustfs_ecstore::api::storage::{ ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext, + POOL_META_NAME as ECSTORE_POOL_META_NAME, }; use rustfs_storage_api as storage_contracts; @@ -36,10 +37,11 @@ pub(crate) mod owner { pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations}; pub(crate) use super::{ - ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET, - EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, - EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext, - EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read, + ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME, + ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, + EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, + EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, + ecstore_local_disk_map_read, }; pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk}; diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index bb2c659b1..1615b9077 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -45,7 +45,7 @@ use tokio::sync::RwLock; use tracing::{debug, error, info, warn}; use uuid::Uuid; -use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET}; +use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, POOL_META_NAME, RUSTFS_META_BUCKET}; #[cfg(test)] pub(crate) struct OutcomeFinishTestHook { @@ -106,7 +106,7 @@ const EVENT_HEAL_ERASURE_SET_STAGE: &str = "heal_erasure_set_stage"; const EVENT_HEAL_ERASURE_SET_RESULT: &str = "heal_erasure_set_result"; /// Heal type -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum HealType { /// Cluster heal Cluster, @@ -209,7 +209,7 @@ impl HealPriority { } /// Heal options -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct HealOptions { /// Scan mode pub scan_mode: HealScanMode, @@ -513,6 +513,11 @@ impl HealTask { } } + #[cfg(test)] + pub(crate) async fn set_execution_elapsed_for_test(&self, elapsed: Duration) { + *self.task_start_instant.write().await = Some(Instant::now() - elapsed); + } + pub(crate) async fn retry_request_with_remaining_timeout(&self) -> Result { let mut request = self.retry_request(); if self.options.timeout.is_some() { @@ -574,6 +579,18 @@ impl HealTask { } } + pub(super) async fn outcome_bucket_incarnation_id(&self, bucket: &str, dry_run: bool) -> Result> { + if dry_run { + return Ok(None); + } + match self.await_with_control(self.storage.bucket_incarnation_id(bucket)).await { + Ok(incarnation_id) => Ok(incarnation_id), + Err(Error::TaskCancelled) => Err(Error::TaskCancelled), + Err(Error::TaskTimeout) => Err(Error::TaskTimeout), + Err(_) => Ok(None), + } + } + fn single_object_identity(&self) -> Option { let (bucket, object, version) = match &self.heal_type { HealType::Object { @@ -597,6 +614,9 @@ impl HealTask { expected: HealObjectIdentity, receipt: Option, ) -> bool { + if self.options.dry_run || self.cancel_token.is_cancelled() { + return false; + } let Some(receipt) = receipt else { return false; }; @@ -631,6 +651,14 @@ impl HealTask { self.batch_failure_recorded.load(Ordering::Acquire) } + pub(crate) async fn batch_failure_is_retryable(&self) -> bool { + self.batch_failure + .read() + .await + .as_ref() + .is_some_and(|failure| failure.failed > 0 && failure.failed == failure.retryable && failure.permanent == 0) + } + pub(crate) async fn record_batch_failure(&self, failure: BatchHealFailure) -> Error { self.batch_failure_recorded.store(true, Ordering::Release); let message = failure.to_string(); diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 5795a743d..cec17886a 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -340,9 +340,88 @@ impl HealTask { return Err(self.record_batch_failure(failure).await); } + if self.options.recreate_missing && !self.options.dry_run { + self.heal_cluster_pool_metadata().await?; + } + Ok(()) } + async fn heal_cluster_pool_metadata(&self) -> Result<()> { + let heal_opts = HealOpts { + recursive: false, + dry_run: self.options.dry_run, + remove: false, + recreate: self.options.recreate_missing, + scan_mode: self.options.scan_mode, + update_parity: self.options.update_parity, + no_lock: self.options.no_lock, + read_repair: false, + pool: self.options.pool_index, + set: self.options.set_index, + }; + + let heal_result = self + .await_with_control(self.storage.heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, &heal_opts)) + .await; + match heal_result { + Ok((result, None)) => { + debug!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket = RUSTFS_META_BUCKET, + object = POOL_META_NAME, + drives_healed = result.drives_healed(), + drives_total = result.drives_reported(), + result = "pool_metadata_ok", + "Heal cluster pool metadata repaired" + ); + self.record_result_item(result).await; + Ok(()) + } + Ok((result, Some(err))) => { + self.record_result_item(result).await; + warn!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket = RUSTFS_META_BUCKET, + object = POOL_META_NAME, + result = "pool_metadata_failed", + error = %err, + "Heal cluster pool metadata failed" + ); + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal cluster pool metadata: {err}"), + }) + } + Err(Error::TaskCancelled) => Err(Error::TaskCancelled), + Err(Error::TaskTimeout) => Err(Error::TaskTimeout), + Err(err) => { + warn!( + target: "rustfs::heal::task", + event = EVENT_HEAL_BUCKET_RESULT, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_TASK, + task_id = %self.id, + bucket = RUSTFS_META_BUCKET, + object = POOL_META_NAME, + result = "pool_metadata_failed", + error = %err, + "Heal cluster pool metadata failed" + ); + Err(Error::TaskExecutionFailed { + message: format!("Failed to heal cluster pool metadata: {err}"), + }) + } + } + } + pub(super) async fn heal_prefix(&self, bucket: &str, prefix: &str) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -403,6 +482,7 @@ impl HealTask { }; for (set_disk_id, heal_opts) in listing_scopes { + let bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, heal_opts.dry_run).await?; let mut continuation_token: Option = None; let mut deferred = DeferredWindow::default(); let mut inline_retry: Option = None; @@ -501,13 +581,15 @@ impl HealTask { let retry_attempt = item.attempt; let mut telemetry_unknown = false; let object = item.name.as_str(); - let identity = + let mut identity = self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set); + identity.bucket_incarnation_id = bucket_incarnation_id; let mut disposition = if heal_opts.dry_run { HealObjectDisposition::DryRunObserved } else { HealObjectDisposition::Unknown }; + let mut recorded_authoritative_outcome = false; let mut detail = None; { let mut progress = self.progress.write().await; @@ -520,23 +602,37 @@ impl HealTask { Some(Error::other("heal object retry age exhausted")) } else { match self - .await_with_control( - self.storage - .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts), - ) + .await_with_control(self.storage.heal_object_with_receipt( + bucket, + object, + item.version_id.as_deref(), + &heal_opts, + )) .await { - Ok((result, None)) => match unavailable_recreate_error(&result, &heal_opts) { - Some(error) => Some(error), - None => { - telemetry_unknown |= !increment_counter(&mut healed); - telemetry_unknown |= - !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX)); - self.record_result_item(result).await; - None + Ok(storage_result) if storage_result.error.is_none() => { + match unavailable_recreate_error(&storage_result.item, &heal_opts) { + Some(error) => Some(error), + None => { + telemetry_unknown |= !increment_counter(&mut healed); + telemetry_unknown |= !add_bytes( + &mut bytes, + u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX), + ); + recorded_authoritative_outcome = self + .record_verified_storage_receipt(identity.clone(), storage_result.receipt) + .await; + self.record_result_item(storage_result.item).await; + None + } } - }, - Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => { + } + Ok(storage_result) + if storage_result + .error + .as_ref() + .is_some_and(|err| is_missing_object_dir_heal_result(object, err)) => + { telemetry_unknown |= !increment_counter(&mut healed); debug!( target: "rustfs::heal::task", @@ -551,7 +647,8 @@ impl HealTask { ); None } - Ok((_, Some(err))) | Err(err) => Some(err), + Ok(storage_result) => storage_result.error, + Err(err) => Some(err), } }; @@ -674,11 +771,13 @@ impl HealTask { continue; } - self.outcome.write().await.record(HealObjectOutcome { - identity, - disposition, - detail, - }); + if !recorded_authoritative_outcome { + self.outcome.write().await.record(HealObjectOutcome { + identity, + disposition, + detail, + }); + } let mut progress = self.progress.write().await; progress.update_object_progress( diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index 76b65a41a..2a6ccc494 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -451,6 +451,11 @@ impl HealTask { self.source, ) .with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone())) + .with_pool_metadata_targets(if self.options.recreate_missing && !self.options.dry_run { + self.heal_endpoints.clone() + } else { + Vec::new() + }) .with_replacement_identity_fence(replacement_target_identities.clone()) .with_mainline_pacer(self.mainline_pacer.clone()); diff --git a/crates/heal/src/heal/task/heal_object.rs b/crates/heal/src/heal/task/heal_object.rs index 9e2c859e7..c7635a79f 100644 --- a/crates/heal/src/heal/task/heal_object.rs +++ b/crates/heal/src/heal/task/heal_object.rs @@ -162,6 +162,9 @@ impl HealTask { pool: self.options.pool_index, set: self.options.set_index, }; + let mut expected_identity = + self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index); + expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?; let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts); let heal_result = if self.source == HealRequestSource::ReadRepair { @@ -266,8 +269,6 @@ impl HealTask { let mut progress = self.progress.write().await; progress.update_object_progress(1, 1, 0, 0, object_size); } - let expected_identity = - self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index); self.record_verified_storage_receipt(expected_identity, storage_result.receipt) .await; self.record_result_item(result).await; diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 01616f087..1e93246c8 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -66,7 +66,7 @@ mod canonical_outcome { assert_eq!(task.get_progress().await.objects_scanned, 2); assert_eq!( storage.heal_object_calls.lock().expect("object calls").as_slice(), - ["object-a", "object-b"] + ["object-a", "object-b", POOL_META_NAME] ); assert_eq!( storage.listing_tokens.lock().expect("listing tokens").as_slice(), @@ -154,6 +154,126 @@ mod canonical_outcome { ); } + #[tokio::test] + async fn bucket_heal_records_matching_positive_storage_receipts() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_receipts: Mutex::new(HashMap::from([ + ( + "object-a".to_string(), + VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]), + ), + ( + "object-b".to_string(), + VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]), + ), + ])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + let task = bucket_task(storage); + + task.execute() + .await + .expect("bucket heal should record verified object receipts"); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.counters.healed, 2); + assert_eq!(outcome.counters.unknown, 0); + assert_eq!(outcome.objects.len(), 2); + assert!(outcome.objects.iter().all(|item| { + item.identity.bucket_incarnation_id == Some(incarnation) && item.disposition == HealObjectDisposition::Repaired + })); + } + + #[tokio::test] + async fn bucket_heal_keeps_repairing_when_bucket_incarnation_is_unavailable() { + let storage = Arc::new(MockStorage { + heal_object_receipts: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([object_receipt( + "object-a", + None, + HealObjectDisposition::Repaired, + Uuid::new_v4(), + )]), + )])), + bucket_incarnation_unavailable: Mutex::new(true), + ..Default::default() + }); + let task = bucket_task(storage.clone()); + + task.execute() + .await + .expect("bucket heal should continue when only proof ownership is unavailable"); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.counters.healed, 0); + assert_eq!(outcome.counters.unknown, 2); + assert!( + outcome + .objects + .iter() + .all(|item| item.disposition == HealObjectDisposition::Unknown) + ); + assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2); + } + + #[tokio::test] + async fn bucket_heal_rejects_stale_receipts_without_double_recording() { + let expected_incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_receipts: Mutex::new(HashMap::from([ + ( + "object-a".to_string(), + VecDeque::from([object_receipt( + "object-a", + None, + HealObjectDisposition::Repaired, + Uuid::new_v4(), + )]), + ), + ( + "object-b".to_string(), + VecDeque::from([object_receipt( + "object-b", + None, + HealObjectDisposition::Repaired, + expected_incarnation, + )]), + ), + ])), + bucket_incarnation_id: Mutex::new(Some(expected_incarnation)), + ..Default::default() + }); + let task = bucket_task(storage); + + task.execute() + .await + .expect("stale bucket receipt should not fail the legacy heal"); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.counters.healed, 1); + assert_eq!(outcome.counters.unknown, 1); + assert_eq!(outcome.objects.len(), 2); + let object_a = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-a") + .expect("stale receipt object outcome"); + assert_eq!(object_a.disposition, HealObjectDisposition::Unknown); + let object_b = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-b") + .expect("matching receipt object outcome"); + assert_eq!(object_b.disposition, HealObjectDisposition::Repaired); + assert_eq!(object_b.identity.bucket_incarnation_id, Some(expected_incarnation)); + } + #[tokio::test(start_paused = true)] async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() { let storage = Arc::new(MockStorage::default()); @@ -242,7 +362,12 @@ mod canonical_outcome { assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1)); let (legacy_summary, legacy_detail) = outcome.legacy_status("finished", None); assert_eq!(legacy_summary, "stopped"); - assert_eq!(legacy_detail.as_deref(), Some("heal traversal completed with errors: 1 failed objects")); + assert_eq!( + legacy_detail.as_deref(), + Some( + "heal traversal completed with errors: 1 failed objects; first failed object bucket-a/object-a (retry_exhausted): Storage error: Lock error: Lock acquisition timeout for resource 'object-a' after 5s" + ) + ); assert_eq!( storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a", "object-b", "object-a", "object-a", "object-a"] @@ -313,6 +438,96 @@ mod canonical_outcome { ); } + #[tokio::test] + async fn mixed_grace_and_repaired_receipt_transfer_only_repaired_responsibility() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_outcomes: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]), + )])), + heal_object_receipts: Mutex::new(HashMap::from([( + "object-b".to_string(), + VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]), + )])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + let task = bucket_task(storage); + + task.execute() + .await + .expect("mixed grace and repaired receipt should complete"); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.healed, 1); + assert_eq!(outcome.counters.skipped, 1); + let deferred = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-a") + .expect("grace object should remain recorded"); + assert!(matches!( + deferred.disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + .. + } + )); + assert_ne!(deferred.disposition, HealObjectDisposition::Repaired); + let repaired = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-b") + .expect("receipt-backed object should be recorded"); + assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation)); + assert_eq!(repaired.disposition, HealObjectDisposition::Repaired); + } + + #[tokio::test] + async fn bucket_heal_records_matching_positive_storage_receipt() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_receipts: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]), + )])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + let task = bucket_task(storage); + + task.execute().await.expect("bucket heal should complete"); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.healed, 1); + assert_eq!(outcome.counters.unknown, 1); + assert_eq!( + outcome + .objects + .iter() + .filter(|item| item.identity.object == "object-a") + .count(), + 1 + ); + let repaired = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-a") + .expect("receipt-backed bucket object should be recorded"); + assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation)); + assert_eq!(repaired.disposition, HealObjectDisposition::Repaired); + let legacy = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-b") + .expect("legacy bucket object should still be recorded"); + assert_eq!(legacy.disposition, HealObjectDisposition::Unknown); + } + #[tokio::test] async fn grace_single_object_is_completed_but_deferred() { let storage = Arc::new(MockStorage { @@ -344,6 +559,18 @@ mod canonical_outcome { .lock() .expect("existence fixture") .insert("object".to_string(), MockObjectExists::TransientSkip("retry later")); + } else { + let incarnation = Uuid::new_v4(); + *storage.bucket_incarnation_id.lock().expect("bucket incarnation") = Some(incarnation); + storage.heal_object_receipts.lock().expect("receipt fixture").insert( + "object".to_string(), + VecDeque::from([object_receipt( + "object", + None, + HealObjectDisposition::VerifiedHealthy, + incarnation, + )]), + ); } let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None); request.options.dry_run = !transient; @@ -1116,6 +1343,9 @@ struct MockStorage { heal_object_outcome: Mutex>, heal_object_outcomes: Mutex>>, heal_object_receipts: Mutex>>, + bucket_incarnation_id: Mutex>, + bucket_incarnation_after_object_heal: Mutex>, + bucket_incarnation_unavailable: Mutex, format_no_heal_required: Mutex, format_error: Mutex>, global_format_calls: Mutex, @@ -1219,14 +1449,19 @@ async fn execute_emits_heal_trace_task_state() { assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed")); } -fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjectDisposition) -> HealObjectReceipt { +fn object_receipt( + object: &str, + version_id: Option<&str>, + disposition: HealObjectDisposition, + bucket_incarnation_id: Uuid, +) -> HealObjectReceipt { HealObjectReceipt { identity: HealObjectIdentity { kind: HealObjectKind::Object, bucket: "bucket-a".to_string(), object: object.to_string(), version_id: version_id.map(ToOwned::to_owned), - bucket_incarnation_id: Some(Uuid::new_v4()), + bucket_incarnation_id: Some(bucket_incarnation_id), pool_index: None, set_index: None, }, @@ -1236,11 +1471,18 @@ fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjec #[tokio::test] async fn object_heal_records_matching_positive_storage_receipt() { + let incarnation = Uuid::new_v4(); let storage = Arc::new(MockStorage { heal_object_receipts: Mutex::new(HashMap::from([( "object-a".to_string(), - VecDeque::from([object_receipt("object-a", Some("version-a"), HealObjectDisposition::Repaired)]), + VecDeque::from([object_receipt( + "object-a", + Some("version-a"), + HealObjectDisposition::Repaired, + incarnation, + )]), )])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), ..Default::default() }); let task = HealTask::from_request( @@ -1261,16 +1503,165 @@ async fn object_heal_records_matching_positive_storage_receipt() { } #[tokio::test] -async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() { +async fn cancelled_object_heal_rejects_matching_positive_storage_receipt() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage::default()); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + task.cancel().await.expect("task cancellation should succeed"); + + let expected = HealObjectIdentity { + kind: HealObjectKind::Object, + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + version_id: Some("version-a".to_string()), + bucket_incarnation_id: Some(incarnation), + pool_index: None, + set_index: None, + }; + let accepted = task + .record_verified_storage_receipt( + expected, + Some(object_receipt( + "object-a", + Some("version-a"), + HealObjectDisposition::Repaired, + incarnation, + )), + ) + .await; + + let outcome = task.get_outcome().await; + assert!(!accepted); + assert_eq!(outcome.counters.healed, 0); + assert!(outcome.objects.is_empty()); +} + +#[tokio::test] +async fn failed_object_heal_rejects_matching_positive_storage_receipt() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithOtherError("commit failed"))), + heal_object_receipts: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([object_receipt( + "object-a", + Some("version-a"), + HealObjectDisposition::Repaired, + incarnation, + )]), + )])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + + let result = task.execute().await; + + let outcome = task.get_outcome().await; + assert!(result.is_err()); + assert_eq!(outcome.counters.healed, 0); + assert_eq!(outcome.counters.unchanged, 0); + assert!(outcome.objects.iter().all(|object| { + !matches!( + object.disposition, + HealObjectDisposition::Repaired + | HealObjectDisposition::VerifiedHealthy + | HealObjectDisposition::AuthoritativelyAbsent + ) + })); +} + +#[tokio::test] +async fn transient_quorum_object_heal_rejects_matching_positive_storage_receipt() { + let incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithReadQuorum)), + heal_object_receipts: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([object_receipt( + "object-a", + Some("version-a"), + HealObjectDisposition::Repaired, + incarnation, + )]), + )])), + bucket_incarnation_id: Mutex::new(Some(incarnation)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + + let result = task.execute().await; + + let outcome = task.get_outcome().await; + assert!(result.is_err()); + assert_eq!(outcome.counters.healed, 0); + assert_eq!(outcome.counters.unchanged, 0); + assert!(outcome.objects.iter().all(|object| { + !matches!( + object.disposition, + HealObjectDisposition::Repaired + | HealObjectDisposition::VerifiedHealthy + | HealObjectDisposition::AuthoritativelyAbsent + ) + })); +} + +#[tokio::test] +async fn object_heal_latches_expected_incarnation_before_repair() { + let original_incarnation = Uuid::new_v4(); + let successor_incarnation = Uuid::new_v4(); let storage = Arc::new(MockStorage { heal_object_receipts: Mutex::new(HashMap::from([( "object-a".to_string(), VecDeque::from([object_receipt( "object-a", - Some("old-version"), - HealObjectDisposition::Repaired, + Some("version-a"), + HealObjectDisposition::VerifiedHealthy, + original_incarnation, )]), )])), + bucket_incarnation_id: Mutex::new(Some(original_incarnation)), + bucket_incarnation_after_object_heal: Mutex::new(Some(successor_incarnation)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())), + storage, + ); + + task.execute().await.expect("mock object heal should complete"); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.unchanged, 1); + assert_eq!(outcome.counters.unknown, 0); + let object = outcome.objects.front().expect("latched receipt should be recorded"); + assert_eq!(object.identity.bucket_incarnation_id, Some(original_incarnation)); + assert_eq!(object.disposition, HealObjectDisposition::VerifiedHealthy); +} + +#[tokio::test] +async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() { + let expected_incarnation = Uuid::new_v4(); + let storage = Arc::new(MockStorage { + heal_object_receipts: Mutex::new(HashMap::from([( + "object-a".to_string(), + VecDeque::from([object_receipt( + "object-a", + Some("version-a"), + HealObjectDisposition::Repaired, + Uuid::new_v4(), + )]), + )])), + bucket_incarnation_id: Mutex::new(Some(expected_incarnation)), ..Default::default() }); let task = HealTask::from_request( @@ -1361,6 +1752,7 @@ enum MockHealObjectOutcome { RetryableLock, RetryableLockTimeout, OkWithOtherError(&'static str), + OkWithReadQuorum, ErrOther(&'static str), DanglingGraceDeferred, UnavailableDrive(DriveState), @@ -1456,6 +1848,13 @@ impl HealStorageAPI for MockStorage { Ok(self.object_exists.lock().unwrap().unwrap_or(true)) } + async fn bucket_incarnation_id(&self, _bucket: &str) -> Result> { + if *self.bucket_incarnation_unavailable.lock().unwrap() { + return Err(Error::Other("bucket incarnation unavailable".to_string())); + } + Ok(*self.bucket_incarnation_id.lock().unwrap()) + } + async fn heal_object( &self, bucket: &str, @@ -1513,6 +1912,13 @@ impl HealStorageAPI for MockStorage { MockHealObjectOutcome::RetryableSlowDown => { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown)))) } + MockHealObjectOutcome::OkWithReadQuorum => Ok(( + HealResultItem::default(), + Some(Error::Storage(EcstoreError::InsufficientReadQuorum( + bucket.to_string(), + object.to_string(), + ))), + )), MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)), MockHealObjectOutcome::OkWithOtherError(message) => Ok((HealResultItem::default(), Some(Error::other(message)))), MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)), @@ -1552,6 +1958,13 @@ impl HealStorageAPI for MockStorage { MockHealObjectOutcome::RetryableSlowDown => { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown)))) } + MockHealObjectOutcome::OkWithReadQuorum => Ok(( + HealResultItem::default(), + Some(Error::Storage(EcstoreError::InsufficientReadQuorum( + bucket.to_string(), + object.to_string(), + ))), + )), }; } if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") { @@ -1566,6 +1979,9 @@ impl HealStorageAPI for MockStorage { return Ok((HealResultItem::default(), Some(Error::Disk(DiskError::FileNotFound)))); } self.healed_objects.lock().unwrap().push(object.to_string()); + if let Some(bucket_incarnation_id) = self.bucket_incarnation_after_object_heal.lock().unwrap().take() { + *self.bucket_incarnation_id.lock().unwrap() = Some(bucket_incarnation_id); + } Ok(( HealResultItem { object_size: 1, @@ -2171,11 +2587,95 @@ async fn test_cluster_heal_visits_bucket_objects() { assert_eq!( storage.healed_objects.lock().unwrap().as_slice(), - ["object-a".to_string(), "object-b".to_string()] + ["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()] ); assert!(matches!(task.get_status().await, HealTaskStatus::Completed)); } +#[tokio::test] +async fn cluster_recreate_heals_pool_metadata_after_user_buckets() { + let storage = Arc::new(MockStorage::default()); + let request = HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + recreate_missing: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage.clone()); + + task.execute() + .await + .expect("cluster recreate heal should include pool metadata"); + + assert_eq!( + storage.heal_object_calls.lock().expect("object calls").as_slice(), + ["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()] + ); + let opts = storage.object_heal_opts.lock().expect("object opts"); + assert!(opts.last().expect("pool metadata opts").recreate); +} + +#[tokio::test] +async fn cluster_recreate_fails_when_pool_metadata_heal_fails() { + let storage = Arc::new(MockStorage::default()); + storage.heal_object_outcomes.lock().expect("object outcomes").insert( + POOL_META_NAME.to_string(), + VecDeque::from([MockHealObjectOutcome::ErrOther("pool metadata missing")]), + ); + let request = HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + recreate_missing: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage.clone()); + + let err = task + .execute() + .await + .expect_err("cluster recreate heal must not hide pool metadata failure"); + + assert!(matches!(err, Error::TaskExecutionFailed { .. })); + assert_eq!( + storage.heal_object_calls.lock().expect("object calls").as_slice(), + ["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()] + ); +} + +#[tokio::test] +async fn cluster_dry_run_does_not_heal_pool_metadata() { + let storage = Arc::new(MockStorage::default()); + let request = HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + dry_run: true, + recreate_missing: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + let task = HealTask::from_request(request, storage.clone()); + + task.execute() + .await + .expect("dry-run cluster heal should preserve existing coverage"); + + assert_eq!( + storage.heal_object_calls.lock().expect("object calls").as_slice(), + ["object-a".to_string(), "object-b".to_string()] + ); +} + #[tokio::test] async fn object_heal_skips_dangling_delete_grace_without_failing_task() { let storage = Arc::new(MockStorage { diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 733701e45..38a3883d7 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -61,10 +61,14 @@ pub fn create_ahm_services_cancel_token() -> CancellationToken { } /// Shutdown all heal services gracefully -pub fn shutdown_ahm_services() { +pub async fn shutdown_ahm_services() -> Result<()> { + if let Some(manager) = get_heal_manager() { + manager.stop().await?; + } if let Some(cancel_token) = GLOBAL_AHM_SERVICES_CANCEL_TOKEN.get() { cancel_token.cancel(); } + Ok(()) } struct HealRuntime { diff --git a/crates/heal/tests/mrf_pipeline_test.rs b/crates/heal/tests/mrf_pipeline_test.rs index 8323995ef..6f0e8081c 100644 --- a/crates/heal/tests/mrf_pipeline_test.rs +++ b/crates/heal/tests/mrf_pipeline_test.rs @@ -22,13 +22,14 @@ //! Under `cargo nextest` each test runs in its own process, which keeps the //! process-global MRF channel singleton safe. -use rustfs_common::mrf_channel::{self, MrfKind}; +use rustfs_common::mrf_channel::{self, MrfIngressResult, MrfKind, MrfScope}; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, mrf_queue, storage::{ECStoreHealStorage, HealStorageAPI}, }; use serial_test::serial; +use sha2::{Digest, Sha256}; #[cfg(unix)] use std::{ fs::{File, OpenOptions}, @@ -48,6 +49,12 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool const META_BUCKET: &str = ".rustfs.sys"; const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin"; const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin"; +const COMMITTED_PAYLOAD_REL: &str = ".heal-mrf-snapshot.0.bin"; +const COMMITTED_MANIFEST_REL: &str = ".heal-mrf-commit.0.bin"; +const COMMITTED_PAYLOAD_RELS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"]; +const COMMITTED_MANIFEST_RELS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"]; +const COMMITTED_MAGIC: &[u8; 8] = b"RFMRFC01"; +const COMMITTED_MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32; async fn heal_env() -> (Vec, Arc) { heal_env_at(None).await @@ -195,6 +202,29 @@ fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) { write_journal_path_to_disks(disk_paths, JOURNAL_REL, data); } +fn committed_manifest(owner: uuid::Uuid, sequence: u64, payload: &[u8]) -> Vec { + let mut manifest = Vec::with_capacity(COMMITTED_MANIFEST_LEN); + manifest.extend_from_slice(COMMITTED_MAGIC); + manifest.push(1); + manifest.extend_from_slice(owner.as_bytes()); + manifest.extend_from_slice(&sequence.to_le_bytes()); + manifest.extend_from_slice( + &u64::try_from(payload.len()) + .expect("fixture payload length fits") + .to_le_bytes(), + ); + manifest.extend_from_slice(&Sha256::digest(payload)); + manifest.extend_from_slice(&Sha256::digest(&manifest)); + assert_eq!(manifest.len(), COMMITTED_MANIFEST_LEN, "committed fixture manifest length"); + manifest +} + +fn write_committed_snapshot_to_disks(disk_paths: &[std::path::PathBuf], sequence: u64, payload: &[u8]) { + let manifest = committed_manifest(uuid::Uuid::new_v4(), sequence, payload); + write_journal_path_to_disks(disk_paths, COMMITTED_PAYLOAD_REL, payload); + write_journal_path_to_disks(disk_paths, COMMITTED_MANIFEST_REL, &manifest); +} + fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path: &str) -> bool { disk_paths .iter() @@ -207,6 +237,71 @@ fn journal_matches_on_all_disks(disk_paths: &[PathBuf], relative_path: &str, exp .all(|path| std::fs::read(path.join(META_BUCKET).join(relative_path)).is_ok_and(|actual| actual == expected)) } +fn journal_contains_on_all_disks(disk_paths: &[PathBuf], relative_path: &str, needle: &[u8]) -> bool { + disk_paths.iter().all(|path| { + std::fs::read(path.join(META_BUCKET).join(relative_path)) + .is_ok_and(|actual| actual.windows(needle.len()).any(|window| window == needle)) + }) +} + +fn journal_contains_on_any_disk(disk_paths: &[PathBuf], relative_path: &str, needle: &[u8]) -> bool { + disk_paths.iter().any(|path| { + std::fs::read(path.join(META_BUCKET).join(relative_path)) + .is_ok_and(|actual| actual.windows(needle.len()).any(|window| window == needle)) + }) +} + +fn committed_payload_contains_on_all_disks(disk_paths: &[PathBuf], needles: &[&[u8]]) -> bool { + disk_paths.iter().all(|path| { + let root = path.join(META_BUCKET); + COMMITTED_PAYLOAD_RELS.into_iter().any(|payload_rel| { + std::fs::read(root.join(payload_rel)).is_ok_and(|payload| { + needles + .iter() + .all(|needle| payload.windows(needle.len()).any(|window| window == *needle)) + }) + }) + }) +} + +fn committed_checkpoint_matches_on_all_disks(disk_paths: &[PathBuf], sequence: u64, expected_payload: &[u8]) -> bool { + disk_paths.iter().all(|path| { + let root = path.join(META_BUCKET); + COMMITTED_PAYLOAD_RELS + .into_iter() + .zip(COMMITTED_MANIFEST_RELS) + .any(|(payload_rel, manifest_rel)| { + let Ok(payload) = std::fs::read(root.join(payload_rel)) else { + return false; + }; + if payload != expected_payload { + return false; + } + let Ok(manifest) = std::fs::read(root.join(manifest_rel)) else { + return false; + }; + if manifest.len() != COMMITTED_MANIFEST_LEN || &manifest[..8] != COMMITTED_MAGIC || manifest[8] != 1 { + return false; + } + let Ok(recorded_sequence) = <[u8; 8]>::try_from(&manifest[25..33]).map(u64::from_le_bytes) else { + return false; + }; + let Ok(recorded_len) = <[u8; 8]>::try_from(&manifest[33..41]).map(u64::from_le_bytes) else { + return false; + }; + let Ok(expected_len) = u64::try_from(expected_payload.len()) else { + return false; + }; + if recorded_sequence != sequence || recorded_len != expected_len { + return false; + } + let payload_digest: [u8; 32] = Sha256::digest(expected_payload).into(); + let manifest_digest: [u8; 32] = Sha256::digest(&manifest[..COMMITTED_MANIFEST_LEN - 32]).into(); + payload_digest.as_slice() == &manifest[41..73] && manifest_digest.as_slice() == &manifest[73..] + }) + }) +} + async fn wait_until(deadline: Duration, mut probe: F) -> bool where F: FnMut() -> Fut, @@ -255,11 +350,12 @@ async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() { } /// A journal left behind by a previous process must be replayed into the -/// manager queue and then removed, and a torn tail must not block replay of -/// the intact records. +/// manager queue, and a torn tail must not block replay of the intact records. +/// The partial-write record keeps the legacy journal as the durable anchor +/// until an exact verified repair proof can discharge it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] -async fn journal_replay_arms_intents_and_deletes_the_file() { +async fn journal_replay_arms_intents_and_retains_unproven_partial_write_anchor() { let (disk_paths, storage) = heal_env().await; // The journal reader resolves disks through the process-local disk map; @@ -285,14 +381,14 @@ async fn journal_replay_arms_intents_and_deletes_the_file() { assert!( disk_paths .iter() - .all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()), - "the journal file must be removed after a successful replay" + .all(|path| Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()), + "partial-write replay must retain the legacy journal until durable proof" ); assert!( disk_paths .iter() .all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()), - "the authoritative journal file must also be removed after replay" + "missing authoritative journal remains absent" ); let snapshot = manager.operations_snapshot().await; @@ -300,6 +396,76 @@ async fn journal_replay_arms_intents_and_deletes_the_file() { assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal"); } +/// A committed checkpoint published by the new two-slot writer is the +/// authoritative startup snapshot. Legacy mirrors are fallback-only and must +/// not be merged with or preferred over the committed epoch. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn committed_snapshot_replay_takes_precedence_over_stale_legacy_mirror() { + let (disk_paths, storage) = heal_env().await; + register_local_disks(&disk_paths, "mrf-committed-replay-test").await; + + let committed = scoped_journal_record(3, "committed-bucket", "committed-object", Some([9u8; 16]), 0, 0, 0); + let stale_legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0); + write_committed_snapshot_to_disks(&disk_paths, 7, &committed); + write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy); + write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy); + + let manager = make_manager(storage); + let replayed = mrf_queue::replay_journal_once(&manager).await; + assert_eq!(replayed, 1, "only the committed snapshot epoch may replay"); + + let snapshot = manager.operations_snapshot().await; + assert_eq!(snapshot.queued_by_source.mrf, 1); + assert_eq!( + snapshot.queued_by_priority.normal, 1, + "the committed partial-write record must replay instead of the stale legacy decode-failure" + ); + assert_eq!( + snapshot.queued_by_priority.urgent, 0, + "stale legacy decode-failure records must not be mixed into committed replay" + ); + assert!( + journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL), + "the committed checkpoint remains until the accepted partial-write has proof" + ); +} + +/// A damaged committed checkpoint is ambiguous: replay must not fall back to +/// older legacy bytes or delete any recovery anchor until another process can +/// publish a valid successor. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn damaged_committed_snapshot_blocks_legacy_fallback_and_retains_anchors() { + let (disk_paths, storage) = heal_env().await; + register_local_disks(&disk_paths, "mrf-damaged-committed-replay-test").await; + + let committed = scoped_journal_record(3, "damaged-committed-bucket", "committed-object", Some([8u8; 16]), 0, 0, 0); + let stale_legacy = journal_record(1, "damaged-legacy-bucket", "legacy-object", None, 0); + write_journal_path_to_disks(&disk_paths, COMMITTED_PAYLOAD_REL, &committed); + let mut manifest = committed_manifest(uuid::Uuid::new_v4(), 9, &committed); + manifest[25] ^= 1; + write_journal_path_to_disks(&disk_paths, COMMITTED_MANIFEST_REL, &manifest); + write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy); + write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy); + + let manager = make_manager(storage); + let replayed = mrf_queue::replay_journal_once(&manager).await; + assert_eq!(replayed, 0, "damaged committed state must fail closed"); + assert_eq!( + manager.operations_snapshot().await.queued_by_source.mrf, + 0, + "stale legacy bytes must not be replayed when committed state is ambiguous" + ); + assert!( + journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL) + && journal_exists_on_all_disks(&disk_paths, COMMITTED_PAYLOAD_REL) + && journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy) + && journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &stale_legacy), + "all recovery anchors must remain after a fail-closed committed read" + ); +} + /// A canonical snapshot and its compatibility mirror may differ after a /// partial flush. Replay must choose the complete canonical epoch instead of /// combining records that never coexisted in memory. @@ -322,21 +488,24 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() { assert_eq!(snapshot.queued_by_source.mrf, 1); assert!( disk_paths.iter().all(|path| { - !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() }), - "replay cleanup must remove both journal paths" + "accepted replay responsibilities remain anchored until a verified repair proof" ); // A scoped-only snapshot is valid during a rollout where no legacy // compatibility mirror was written. Missing legacy files must not leave // the runtime in a permanent cleanup-retry state. + let (disk_paths, storage) = heal_env().await; + register_local_disks(&disk_paths, "mrf-scoped-authoritative-test").await; + let manager = make_manager(storage); let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0); write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only); assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1); assert!(disk_paths.iter().all(|path| { !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() })); let scoped_v2 = scoped_journal_record(1, "scoped-v2-bucket", "scoped-v2-object", None, 0, 3, 7); @@ -350,12 +519,12 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() { ); assert_eq!( manager.operations_snapshot().await.queued_by_source.mrf, - 3, - "only the three authoritative/scoped-only epochs should have reached the manager" + 2, + "only the scoped-only and scoped-v2 authoritative epochs should have reached the manager" ); assert!(disk_paths.iter().all(|path| { - !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() })); } @@ -400,10 +569,13 @@ async fn authoritative_journal_replay_preserves_kind_and_scope_identity() { snapshot.queued_by_priority.urgent, 1, "decode-failure repair must not merge with object repair responsibility" ); - assert!(disk_paths.iter().all(|path| { - !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() - })); + assert!( + disk_paths.iter().all(|path| { + Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + }), + "partial-write responsibilities keep both replay anchors until proof" + ); } /// If replay reaches a full heal-manager queue, the old journal remains the @@ -470,6 +642,73 @@ async fn journal_replay_retains_file_when_manager_is_full() { ); } +/// Rollback mirrors are for v1 readers only: the committed and scoped +/// snapshots remain authoritative, while the legacy journal omits scoped-only +/// records that an older binary cannot represent safely. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn rollback_legacy_mirror_persists_only_v1_compatible_records() { + let (disk_paths, storage) = heal_env().await; + register_local_disks(&disk_paths, "mrf-rollback-mirror-test").await; + + let manager = Arc::new(HealManager::new( + storage, + Some(HealConfig { + queue_size: 0, + heal_interval: Duration::from_secs(3600), + enable_auto_heal: false, + ..Default::default() + }), + )); + mrf_queue::spawn_mrf_consumer(manager.clone()); + + let scoped_only = b"rollback-scoped-only-object"; + let v1_compatible = b"rollback-v1-compatible-object"; + assert_eq!( + mrf_channel::try_send_mrf_intent_typed( + MrfKind::PartialWrite, + "rollback-bucket", + std::str::from_utf8(scoped_only).expect("fixture object is UTF-8"), + None, + Some(MrfScope { + pool_index: 3, + set_index: 7, + }), + ), + MrfIngressResult::Enqueued, + "scoped-only intent should be accepted by the live consumer" + ); + assert_eq!( + mrf_channel::try_send_mrf_intent_typed( + MrfKind::PartialWrite, + "rollback-bucket", + std::str::from_utf8(v1_compatible).expect("fixture object is UTF-8"), + None, + None, + ), + MrfIngressResult::Enqueued, + "v1-compatible intent should be accepted by the live consumer" + ); + + let flushed = wait_until(Duration::from_secs(10), || async { + committed_payload_contains_on_all_disks(&disk_paths, &[scoped_only, v1_compatible]) + && journal_contains_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, scoped_only) + && journal_contains_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, v1_compatible) + && journal_contains_on_all_disks(&disk_paths, JOURNAL_REL, v1_compatible) + && !journal_contains_on_any_disk(&disk_paths, JOURNAL_REL, scoped_only) + }) + .await; + assert!( + flushed, + "runtime flush must persist rollback-safe mirrors without leaking scoped-only records into the legacy journal" + ); + assert_eq!( + manager.operations_snapshot().await.queued_by_source.mrf, + 0, + "zero-capacity manager keeps both intents in the MRF runtime so the persisted snapshot is observable" + ); +} + #[test] fn mrf_journal_child_process_fixture() { let Ok(root) = std::env::var("RUSTFS_MRF_REPLAY_CHILD_ROOT") else { @@ -524,13 +763,14 @@ fn mrf_successor_flush_child_process_fixture() { let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2); let flushed = wait_until(Duration::from_secs(10), || async { manager.operations_snapshot().await.queued_by_source.mrf == 1 + && committed_checkpoint_matches_on_all_disks(&disk_paths, 2, &expected_successor) && journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor) && journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor) }) .await; assert!( flushed, - "child process must publish the pending successor snapshot before the delete phase" + "child process must publish the committed pending successor before the delete phase" ); }); std::process::exit(78); @@ -571,13 +811,14 @@ fn mrf_successor_flush_waiting_child_process_fixture() { let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2); let flushed = wait_until(Duration::from_secs(10), || async { manager.operations_snapshot().await.queued_by_source.mrf == 1 + && committed_checkpoint_matches_on_all_disks(&disk_paths, 2, &expected_successor) && journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor) && journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor) }) .await; assert!( flushed, - "child process must publish the pending successor snapshot before it can be killed" + "child process must publish the committed pending successor before it can be killed" ); std::fs::write(&ready_path, b"ready").expect("write ready marker"); loop { @@ -698,10 +939,10 @@ async fn journal_replay_survives_successor_flush_before_delete() { ); assert!( disk_paths.iter().all(|path| { - !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() }), - "a fully consumed successor snapshot may be deleted after restart replay" + "the accepted successor remains anchored until a verified repair proof" ); } @@ -751,10 +992,10 @@ async fn journal_replay_survives_service_kill_after_successor_flush() { ); assert!( disk_paths.iter().all(|path| { - !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() }), - "a fully consumed successor snapshot may be deleted after service-kill restart replay" + "the accepted successor remains anchored until a verified repair proof after service-kill restart" ); } @@ -814,9 +1055,9 @@ async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_bef ); assert!( disk_paths.iter().all(|path| { - !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() - && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() }), - "a fully consumed authoritative successor may clean both epochs after restart replay" + "the accepted authoritative successor remains anchored until a verified repair proof" ); } diff --git a/crates/kms/src/backends/aws.rs b/crates/kms/src/backends/aws.rs index 87ee53ae7..5ffcb8dd0 100644 --- a/crates/kms/src/backends/aws.rs +++ b/crates/kms/src/backends/aws.rs @@ -819,7 +819,9 @@ impl KmsBackend for AwsKmsBackend { .with_rotate(true) .with_enable_disable(true) .with_schedule_deletion(true) - .with_versioning(true) + // AWS KMS exposes rotation state but does not enumerate key + // versions through this backend's API contract. + .with_versioning(false) .with_physical_delete(false) .with_production_supported(true) } diff --git a/crates/kms/src/backends/scripted_vault.rs b/crates/kms/src/backends/scripted_vault.rs index 2450c7625..f6882dbfb 100644 --- a/crates/kms/src/backends/scripted_vault.rs +++ b/crates/kms/src/backends/scripted_vault.rs @@ -74,6 +74,39 @@ impl ScriptedResponse { } } +/// The `auth/token/lookup-self` answer every scripted Vault serves for free. +/// +/// A Vault client now probes its token's remaining lifetime at login +/// (backlog#2369 P3), which is credential plumbing rather than the protocol any +/// of these tests is scripting. Answering it out of band keeps every existing +/// script meaningful: `ttl` 0 is Vault's "this token does not expire", so the +/// probe changes nothing about how a scripted test behaves. +pub(crate) fn token_lookup_self_response() -> String { + serde_json::json!({ + "data": { + "accessor": "scripted-accessor", + "creation_time": 1_700_000_000u64, + "creation_ttl": 0, + "display_name": "token", + "entity_id": "", + "explicit_max_ttl": 0, + "id": "scripted-token", + "num_uses": 0, + "orphan": true, + "path": "auth/token/create", + "policies": ["default"], + "renewable": false, + "ttl": 0 + } + }) + .to_string() +} + +/// Whether a recorded request line addresses the token self-lookup. +pub(crate) fn is_token_lookup_self(request_line: &str) -> bool { + request_line.contains("/v1/auth/token/lookup-self") +} + /// A scripted stand-in Vault listening on a loopback port. pub(crate) struct ScriptedVault { /// Base address (`http://127.0.0.1:port`) to point a Vault client at. @@ -102,6 +135,19 @@ impl ScriptedVault { let Some((request_line, body, mut stream)) = read_request(stream).await else { continue; }; + if is_token_lookup_self(&request_line) { + // Served out of band so the credential probe does not + // consume a scripted response meant for the protocol under + // test, and is not recorded as one of its requests. + let body = token_lookup_self_response(); + let payload = format!( + "HTTP/1.1 200 Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len(), + ); + let _ = stream.write_all(payload.as_bytes()).await; + let _ = stream.shutdown().await; + continue; + } recorded .lock() .expect("scripted vault request log poisoned") @@ -155,6 +201,19 @@ impl ScriptedVault { let Some((request_line, body, stream)) = read_request(stream).await else { return; }; + if is_token_lookup_self(&request_line) { + // Credential plumbing, not part of the KV2 protocol + // this responder models; see token_lookup_self_response. + write_response( + stream, + ScriptedResponse::Http { + status: 200, + body: token_lookup_self_response(), + }, + ) + .await; + return; + } recorded .lock() .expect("scripted vault request log poisoned") diff --git a/crates/kms/src/backends/snapshots/rustfs_kms__backends__aws__tests__aws_backend_capabilities.snap b/crates/kms/src/backends/snapshots/rustfs_kms__backends__aws__tests__aws_backend_capabilities.snap index 59b119950..9803d4a03 100644 --- a/crates/kms/src/backends/snapshots/rustfs_kms__backends__aws__tests__aws_backend_capabilities.snap +++ b/crates/kms/src/backends/snapshots/rustfs_kms__backends__aws__tests__aws_backend_capabilities.snap @@ -13,5 +13,5 @@ expression: capabilities_snapshot(backend.capabilities()) "rotate": true, "schedule_deletion": true, "update_key_metadata": false, - "versioning": true + "versioning": false } diff --git a/crates/kms/src/backends/vault_credentials.rs b/crates/kms/src/backends/vault_credentials.rs index a9b95703d..a79269cd5 100644 --- a/crates/kms/src/backends/vault_credentials.rs +++ b/crates/kms/src/backends/vault_credentials.rs @@ -114,7 +114,7 @@ impl fmt::Debug for SecretString { } /// Expiry attributes of a lease-bound token. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct LeaseInfo { /// Time-to-live granted at issue or renewal. pub(crate) ttl: Duration, @@ -207,31 +207,102 @@ pub(crate) trait TokenSource: fmt::Debug + Send + Sync { } /// Token source for [`VaultAuthMethod::Token`]: always yields the token fixed -/// at configuration time. The token carries no lease, so it is never renewed -/// and never expires from the provider's point of view. +/// at configuration time. +/// +/// The token itself is never re-issued, but it usually still expires: +/// `vault token create` defaults to a 768-hour TTL. Hard-coding "no lease" +/// here left the renewal task unstarted and published no remaining-TTL gauge, +/// so a healthy-looking cluster turned every KMS call into a 403 a month later +/// and could only be recovered by a restart or a reconfigure (backlog#2369 P3). +/// The source therefore asks Vault what it is holding, once per client +/// generation, and lets the existing renewal loop take over whenever the answer +/// carries a TTL. +/// Map a `lookup-self` answer onto a lease. +/// +/// A zero TTL is Vault's answer for a token that never expires (root and +/// periodic-root tokens), which keeps the pre-probe behaviour exactly: no +/// lease, no renewal task, no expiry gate. A response that omits `renewable` +/// is treated as not renewable, so the renewal loop falls back to re-reading +/// the remaining TTL instead of assuming it can extend it. +fn static_token_lease(ttl_secs: u64, renewable: Option) -> Option { + (ttl_secs > 0).then_some(LeaseInfo { + ttl: Duration::from_secs(ttl_secs), + renewable: renewable.unwrap_or(false), + }) +} + pub(crate) struct StaticToken { token: TokenLease, + /// Client authenticated with the configured token, used only for + /// `lookup-self`. Per-generation renewals use the generation's own client. + lookup_client: VaultClient, } impl StaticToken { - pub(crate) fn new(token: String) -> Self { - Self { + pub(crate) fn new(settings: &VaultConnectionSettings, token: String) -> Result { + let lookup_client = settings.build_client(&token)?; + Ok(Self { token: TokenLease::new(token, None), - } + lookup_client, + }) } } #[async_trait] impl TokenSource for StaticToken { async fn acquire(&self) -> AttemptResult { - Ok(self.token.clone()) + // A lookup failure must not fail the login. The token itself may well + // be valid: a policy can omit `lookup-self`, and Vault may simply be + // unreachable for the moment. Failing here would take down deployments + // that work today, so the probe degrades to the pre-probe behaviour — + // no lease, no renewal — and says so loudly instead. + let lease = match vaultrs::token::lookup_self(&self.lookup_client).await { + Ok(lookup) => static_token_lease(lookup.ttl, lookup.renewable), + Err(error) => { + warn!( + event = "vault_static_token_lookup_failed", + error = %error, + "Could not read the configured Vault token's remaining lifetime, so it will not be \ + renewed and its expiry will not be tracked. Grant the token `lookup-self` (Vault's \ + default policy does) or switch to AppRole, Kubernetes or an agent-managed token file" + ); + None + } + }; + + if let Some(lease) = lease + && !lease.renewable + { + warn!( + event = "vault_static_token_not_renewable", + ttl_secs = lease.ttl.as_secs(), + "The configured Vault token expires and cannot be renewed; RustFS will fail closed as it \ + approaches expiry. Switch to AppRole, Kubernetes or an agent-managed token file, or \ + reconfigure with a fresh token before it lapses" + ); + } + + Ok(TokenLease::new(self.token.expose().to_string(), lease)) + } + + async fn renew(&self, client: &VaultClient) -> AttemptResult { + // Vault refuses renew-self on a non-renewable token; the renewal loop + // then falls back to `acquire`, which re-reads the remaining TTL and + // keeps the gauge honest until the fail-closed window is reached. + let auth = vaultrs::token::renew_self(client, None) + .await + .map_err(|error| attempt_error("token renewal", error))?; + Ok(TokenLease::from_auth(auth)) } } impl fmt::Debug for StaticToken { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // TokenLease::fmt already redacts the token value. - f.debug_struct("StaticToken").field("token", &self.token).finish() + // TokenLease::fmt already redacts the token value; VaultClient embeds + // its settings, including the token, so it must stay out of Debug. + f.debug_struct("StaticToken") + .field("token", &self.token) + .finish_non_exhaustive() } } @@ -541,7 +612,7 @@ pub(crate) fn token_source_for( settings: &VaultConnectionSettings, ) -> Result> { match auth_method { - VaultAuthMethod::Token { token } => Ok(Box::new(StaticToken::new(token.clone()))), + VaultAuthMethod::Token { token } => Ok(Box::new(StaticToken::new(settings, token.clone())?)), VaultAuthMethod::AppRole { role_id, secret_id, @@ -1201,14 +1272,22 @@ mod tests { (Arc::new(provider), state) } + /// A provider whose token reports no expiry, which is what `lookup-self` + /// answers for a root or periodic-root token. Scripted rather than backed + /// by [`StaticToken`] because the real source now asks Vault what it holds. async fn static_provider() -> VaultCredentialProvider { VaultCredentialProvider::new( test_settings(), - Box::new(StaticToken::new(TEST_TOKEN.to_string())), + Box::new(ScriptedSource { + state: Arc::new(ScriptedState::default()), + ttl: Duration::ZERO, + renewable: false, + login_delay: Duration::ZERO, + }), test_policy(Duration::from_secs(10), Duration::from_secs(5)), ) .await - .expect("static provider must build without a live Vault") + .expect("a token without an expiry must build without a live Vault") } #[tokio::test] @@ -1231,20 +1310,89 @@ mod tests { assert!(provider.spawn_renewal_task().is_none(), "a token without a lease has nothing to renew"); } - #[tokio::test] - async fn test_static_token_source_yields_configured_token() { - let settings = test_settings(); - let source = token_source_for( + #[test] + fn test_static_token_source_builds_without_contacting_vault() { + token_source_for( &VaultAuthMethod::Token { token: TEST_TOKEN.to_string(), }, - &settings, + &test_settings(), ) .expect("token auth must map to a source"); + } - let lease = source.acquire().await.expect("static acquire cannot fail"); - assert_eq!(lease.expose(), TEST_TOKEN); - assert!(lease.lease_info().is_none(), "static tokens must not carry a lease"); + /// A pending acquisition cannot take the returned-error fallback: the + /// outer login policy must cut it off before publishing a client. + #[tokio::test(start_paused = true)] + async fn test_stalled_initial_login_is_bounded_by_the_attempt_timeout() { + let state = Arc::new(ScriptedState::default()); + let source = ScriptedSource { + state: state.clone(), + ttl: Duration::ZERO, + renewable: false, + login_delay: Duration::from_secs(60), + }; + let policy = test_policy(Duration::from_secs(10), Duration::from_secs(5)); + let attempt_timeout = policy.retry.attempt_timeout; + let started = Instant::now(); + let result = VaultCredentialProvider::new(test_settings(), Box::new(source), policy).await; + assert!( + matches!(&result, Err(KmsError::OperationTimedOut { message }) if message.starts_with("vault_login attempt 1 timed out")), + "a stalled login must return its typed timeout without publishing a client" + ); + assert_eq!( + started.elapsed(), + attempt_timeout, + "login must consume exactly one virtual attempt budget" + ); + assert_eq!(state.login_calls.load(Ordering::SeqCst), 0, "the acquisition must not complete"); + assert_eq!( + state.renew_calls.load(Ordering::SeqCst), + 0, + "failed initialization must not start renewal" + ); + } + + /// backlog#2369 P3: `vault token create` defaults to a 768-hour TTL, so + /// hard-coding "no lease" for token auth left the renewal task unstarted + /// and turned a healthy cluster into one that answers 403 a month later. + /// The lease now comes from what Vault reports. + #[test] + fn static_token_lease_follows_what_vault_reports() { + assert_eq!( + static_token_lease(0, Some(true)), + None, + "a token Vault reports as non-expiring must keep behaving as one" + ); + assert_eq!( + static_token_lease(0, None), + None, + "a non-expiring token stays non-expiring whatever renewable says" + ); + assert_eq!( + static_token_lease(2_764_800, Some(true)), + Some(LeaseInfo { + ttl: Duration::from_secs(2_764_800), + renewable: true, + }), + "the default 768-hour token must be tracked and renewed" + ); + assert_eq!( + static_token_lease(3_600, Some(false)), + Some(LeaseInfo { + ttl: Duration::from_secs(3_600), + renewable: false, + }), + "an expiring token that cannot be renewed still needs its expiry tracked" + ); + assert_eq!( + static_token_lease(3_600, None), + Some(LeaseInfo { + ttl: Duration::from_secs(3_600), + renewable: false, + }), + "an omitted renewable flag must not be read as renewable" + ); } #[tokio::test] @@ -1759,7 +1907,7 @@ mod tests { renewable: true, }), ); - let static_source = StaticToken::new(TEST_TOKEN.to_string()); + let static_source = StaticToken::new(&test_settings(), TEST_TOKEN.to_string()).expect("static source"); let approle_source = AppRoleLogin::new( &test_settings(), "approle".to_string(), diff --git a/crates/kms/tests/vault_fault_injection.rs b/crates/kms/tests/vault_fault_injection.rs index b84c9c5d7..bc772da2c 100644 --- a/crates/kms/tests/vault_fault_injection.rs +++ b/crates/kms/tests/vault_fault_injection.rs @@ -14,8 +14,7 @@ //! Fault-injection matrix for the Vault backend operation policy. //! -//! Offline cases run against locally injected transport faults (a listener -//! that never responds) — deterministic, no external +//! Offline cases run against locally injected HTTP and transport faults — no external //! dependencies. Real-Vault cases are `#[ignore]`d and need a dev Vault //! (default `http://127.0.0.1:8200`, override with `RUSTFS_KMS_VAULT_ADDR`). //! @@ -38,9 +37,100 @@ use rustfs_kms::backends::vault::VaultKmsBackend; use rustfs_kms::{ BackendConfig, DescribeKeyRequest, KmsBackend as KmsBackendKind, KmsConfig, KmsError, VaultAuthMethod, VaultConfig, }; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total"; const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total"; +const LOGIN: &str = "vault_login"; +const READ_KEY: &str = "vault_kv2_read_key"; +const LOOKUP_REQUEST: &str = "GET /v1/auth/token/lookup-self HTTP/1.1"; + +/// Unlike the unit-test scripted Vault, this fixture records the credential +/// probe too and can fail it independently of the subsequent key request. +/// `None` parks a connection without responding; extra requests receive 599. +struct FaultVault { + address: String, + requests: mpsc::UnboundedReceiver, + task: JoinHandle<()>, +} + +impl FaultVault { + async fn serve(responses: Vec>) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fault-injection Vault"); + let address = format!("http://{}", listener.local_addr().expect("fault-injection Vault address")); + let (recorded, requests) = mpsc::unbounded_channel(); + let task = tokio::spawn(async move { + let mut responses = responses.into_iter(); + let mut parked = Vec::new(); + loop { + let (stream, _) = listener.accept().await.expect("accept Vault request"); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + assert_ne!(stream.read_line(&mut line).await.expect("read request line"), 0); + recorded.send(line.trim_end().to_string()).expect("record Vault request"); + loop { + line.clear(); + assert_ne!(stream.read_line(&mut line).await.expect("read request header"), 0); + if line == "\r\n" { + break; + } + } + let mut stream = stream.into_inner(); + let response = responses + .next() + .unwrap_or_else(|| Some((599, serde_json::json!({"errors": ["unexpected Vault request"]})))); + if let Some((status, body)) = response { + let body = body.to_string(); + let response = format!( + "HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.expect("write Vault response"); + stream.shutdown().await.expect("close Vault response"); + } else { + parked.push(stream); + } + } + }); + Self { address, requests, task } + } + + async fn finish(&mut self) { + assert!(self.requests.try_recv().is_err(), "no unexpected requests may remain"); + self.task.abort(); + let error = (&mut self.task).await.expect_err("fault server runs until aborted"); + assert!(error.is_cancelled(), "fault server must not panic: {error}"); + } +} + +impl Drop for FaultVault { + fn drop(&mut self) { + self.task.abort(); + } +} + +fn healthy_token_lookup() -> serde_json::Value { + serde_json::json!({ + "data": { + "accessor": "fault-injection-accessor", + "creation_time": 1_700_000_000u64, + "creation_ttl": 0, + "display_name": "token", + "entity_id": "", + "explicit_max_ttl": 0, + "id": "unused", + "num_uses": 0, + "orphan": true, + "path": "auth/token/create", + "policies": ["default"], + "renewable": false, + "ttl": 0 + } + }) +} fn vault_config(address: &str, token: &str) -> VaultConfig { VaultConfig { @@ -125,39 +215,112 @@ fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) fn stalled_connection_is_cut_off_by_the_attempt_timeout() { let snapshot = record_metrics(|| { Box::pin(async move { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + let mut vault = FaultVault::serve(vec![Some((200, healthy_token_lookup())), None]).await; + let attempt_timeout = Duration::from_millis(250); + let client = VaultKmsBackend::new(kms_config(vault_config(&vault.address, "unused"), attempt_timeout, 1)) .await - .expect("bind stall listener"); - let address = format!("http://{}", listener.local_addr().expect("stall listener addr")); - // Accept and park every connection without ever responding. - tokio::spawn(async move { - let mut parked = Vec::new(); - loop { - let Ok((socket, _)) = listener.accept().await else { return }; - parked.push(socket); - } - }); + .expect("the token lookup must succeed before injecting the stalled key read"); + assert_eq!(vault.requests.try_recv().as_deref(), Ok(LOOKUP_REQUEST)); - let client = VaultKmsBackend::new(kms_config(vault_config(&address, "unused"), Duration::from_millis(250), 1)) - .await - .expect("client construction performs no network calls"); - let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled")) + let read = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-stalled")); + tokio::pin!(read); + tokio::select! { + request = vault.requests.recv() => assert_eq!( + request.as_deref(), + Some("GET /v1/secret/data/rustfs/kms/fault-injection/fault-injection-stalled? HTTP/1.1") + ), + result = &mut read => panic!("the key request must reach the stall listener: {result:?}"), + } + // Pause only after real loopback I/O reaches the intended request; + // otherwise auto-advancing time could expire the login instead. + tokio::time::pause(); + let stalled_at = tokio::time::Instant::now(); + // Tokio rounds timer deadlines up to the next millisecond. + let virtual_step = attempt_timeout + Duration::from_millis(1); + tokio::time::advance(virtual_step).await; + let error = tokio::time::timeout(Duration::from_secs(1), read) .await + .expect("the attempt timer must resolve without further network activity") .expect_err("a stalled request must be cut off by the attempt timeout"); + assert_eq!( + stalled_at.elapsed(), + virtual_step, + "the read must resolve within the attempt budget plus one timer tick" + ); assert!( matches!(error, KmsError::OperationTimedOut { .. } | KmsError::BackendError { .. }), "got {error:?}" ); + vault.finish().await; }) }); // The policy timer reports attempt_timeout; the client-level HTTP timeout // surfaces as a connection-class failure. Either way it is exactly one // attempt that was cut off. - let cut_off = counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "attempt_timeout")]) - + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_conn")]); + let cut_off = counter_value( + &snapshot, + ATTEMPT_FAILURES_TOTAL, + &[("operation", READ_KEY), ("error_class", "attempt_timeout")], + ) + counter_value( + &snapshot, + ATTEMPT_FAILURES_TOTAL, + &[("operation", READ_KEY), ("error_class", "retryable_conn")], + ); assert_eq!(cut_off, 1, "the single budgeted attempt must be cut off by a timeout"); - assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]), 1); + assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", LOGIN)]), 0); + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", LOGIN), ("outcome", "success")]), + 1 + ); + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "budget_exhausted")]), + 1 + ); +} + +/// A returned lookup error degrades lease discovery, not Vault authorization: +/// the subsequent forbidden key read must still fail once, without retrying. +#[test] +fn token_lookup_errors_do_not_bypass_key_authorization() { + for lookup_status in [403, 503] { + let snapshot = record_metrics(|| { + Box::pin(async move { + let mut vault = FaultVault::serve(vec![ + Some((lookup_status, serde_json::json!({"errors": ["token lookup unavailable"]}))), + Some((403, serde_json::json!({"errors": ["permission denied"]}))), + ]) + .await; + let client = VaultKmsBackend::new(kms_config(vault_config(&vault.address, "unused"), Duration::from_secs(5), 3)) + .await + .expect("a returned token lookup error must preserve static-token fallback"); + assert_eq!(vault.requests.try_recv().as_deref(), Ok(LOOKUP_REQUEST)); + let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden")) + .await + .expect_err("lease discovery fallback must not authorize a forbidden key read"); + assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}"); + assert_eq!( + vault.requests.try_recv().as_deref(), + Ok("GET /v1/secret/data/rustfs/kms/fault-injection/fault-injection-forbidden? HTTP/1.1") + ); + vault.finish().await; + }) + }); + assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", LOGIN)]), 0); + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", LOGIN), ("outcome", "success")]), + 1 + ); + assert_eq!(counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY)]), 1); + assert_eq!( + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]), + 1 + ); + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]), + 1 + ); + } } fn real_vault_address() -> String { @@ -174,7 +337,7 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() { let config = vault_config(&real_vault_address(), "fault-injection-invalid-token"); let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3)) .await - .expect("client construction performs no network calls"); + .expect("a returned token lookup error must preserve static-token fallback"); let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-forbidden")) .await .expect_err("an invalid token must be rejected"); @@ -183,13 +346,20 @@ fn real_vault_invalid_token_is_fatal_and_never_retried() { }); assert_eq!( - counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]), + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]), 1, "a 403 must be observed by exactly one attempt" ); - assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1); assert_eq!( - counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "retryable_status")]), + counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]), + 1 + ); + assert_eq!( + counter_value( + &snapshot, + ATTEMPT_FAILURES_TOTAL, + &[("operation", READ_KEY), ("error_class", "retryable_status")] + ), 0, "an auth failure must never be classified as retryable" ); @@ -207,7 +377,7 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() { let config = vault_config(&real_vault_address(), &token); let client = VaultKmsBackend::new(kms_config(config, Duration::from_secs(5), 3)) .await - .expect("client construction performs no network calls"); + .expect("static-token initialization must complete against the running Vault"); let error = KmsBackendTrait::describe_key(&client, describe_key_request("fault-injection-definitely-missing")) .await .expect_err("a missing key must resolve to key-not-found"); @@ -216,9 +386,12 @@ fn real_vault_missing_key_is_resolved_in_one_attempt() { }); assert_eq!( - counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("error_class", "fatal")]), + counter_value(&snapshot, ATTEMPT_FAILURES_TOTAL, &[("operation", READ_KEY), ("error_class", "fatal")]), 1, "a 404 must be observed by exactly one attempt" ); - assert_eq!(counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "fatal")]), 1); + assert_eq!( + counter_value(&snapshot, OPERATIONS_TOTAL, &[("operation", READ_KEY), ("outcome", "fatal")]), + 1 + ); } diff --git a/crates/madmin/src/client.rs b/crates/madmin/src/client.rs index f76c1048f..bd68be97e 100644 --- a/crates/madmin/src/client.rs +++ b/crates/madmin/src/client.rs @@ -263,6 +263,14 @@ pub struct ScannerUsageRecoveryIntentResponse { pub mode: String, pub intent_id: String, pub state: String, + #[serde(default)] + pub actor_sha256: Option, + #[serde(default)] + pub idempotency_key_sha256: Option, + #[serde(default)] + pub request_sha256: Option, + #[serde(default)] + pub accepted_at_unix_secs: Option, #[serde(flatten)] pub extra: serde_json::Map, } @@ -869,6 +877,10 @@ mod tests { "mode": "full-rebuild", "intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "state": "accepted", + "actor_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "idempotency_key_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "request_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "accepted_at_unix_secs": 7, "future": {"worker": "pending"} })) .unwrap(); @@ -877,7 +889,33 @@ mod tests { assert_eq!(intent.mode, "full-rebuild"); assert_eq!(intent.intent_id, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); assert_eq!(intent.state, "accepted"); + assert_eq!( + intent.actor_sha256.as_deref(), + Some("1111111111111111111111111111111111111111111111111111111111111111") + ); + assert_eq!( + intent.idempotency_key_sha256.as_deref(), + Some("2222222222222222222222222222222222222222222222222222222222222222") + ); + assert_eq!( + intent.request_sha256.as_deref(), + Some("3333333333333333333333333333333333333333333333333333333333333333") + ); + assert_eq!(intent.accepted_at_unix_secs, Some(7)); assert_eq!(intent.extra["future"]["worker"], "pending"); + + let legacy_intent: ScannerUsageRecoveryIntentResponse = serde_json::from_value(json!({ + "status": "accepted", + "action": "usage-full-rebuild", + "mode": "full-rebuild", + "intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "state": "accepted" + })) + .unwrap(); + assert!(legacy_intent.actor_sha256.is_none()); + assert!(legacy_intent.idempotency_key_sha256.is_none()); + assert!(legacy_intent.request_sha256.is_none()); + assert!(legacy_intent.accepted_at_unix_secs.is_none()); } #[test] @@ -971,7 +1009,7 @@ mod tests { #[tokio::test] async fn scanner_usage_async_reset_posts_explicit_intent_contract() { let server = TestServer::spawn( - r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted"}"#, + r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted","actor_sha256":"1111111111111111111111111111111111111111111111111111111111111111","idempotency_key_sha256":"2222222222222222222222222222222222222222222222222222222222222222","request_sha256":"3333333333333333333333333333333333333333333333333333333333333333","accepted_at_unix_secs":7}"#, 202, ) .await; @@ -985,6 +1023,7 @@ mod tests { assert_eq!(accepted.status, "accepted"); assert_eq!(accepted.mode, "full-rebuild"); assert_eq!(accepted.state, "accepted"); + assert_eq!(accepted.accepted_at_unix_secs, Some(7)); let request = server.recorded(); assert_eq!(request.method, "POST"); assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset"); diff --git a/crates/madmin/tests/fixtures/heal-outcome-v3.json b/crates/madmin/tests/fixtures/heal-outcome-v3.json index 9f634af80..21989d323 100644 --- a/crates/madmin/tests/fixtures/heal-outcome-v3.json +++ b/crates/madmin/tests/fixtures/heal-outcome-v3.json @@ -98,7 +98,7 @@ "cliExit": 1, "response": { "summary": "stopped", - "detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated", + "detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated", "startTime": "2026-01-01T00:00:00Z", "settings": { "recursive": true, @@ -274,7 +274,7 @@ "cliExit": 1, "response": { "summary": "stopped", - "detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated", + "detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated", "startTime": "2026-01-01T00:00:00Z", "settings": { "recursive": true, diff --git a/crates/obs/README.md b/crates/obs/README.md index 38cf734bd..43ab1abc3 100644 --- a/crates/obs/README.md +++ b/crates/obs/README.md @@ -122,7 +122,11 @@ All configuration is read from environment variables at startup. | `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `false` | Toggle profiling export | | `RUSTFS_OBS_USE_STDOUT` | `false` | Mirror all signals to stdout alongside OTLP | | `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` | -| `RUSTFS_OBS_METER_INTERVAL` | `15` | Metrics export interval (seconds) | +| `RUSTFS_OBS_METER_INTERVAL` | `30` | Metrics export interval (seconds) | + +The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and per-bucket metrics every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides. + +With OTLP → Collector → Prometheus, the time from a failure to an alert also includes underlying health detection, collection and export work, transport and Collector batching, Prometheus `scrape_interval` and `evaluation_interval`, and any alert rule `for` duration. A shorter scrape interval cannot refresh a value that RustFS has not yet collected and exported. These periodic intervals do not guarantee a fixed detection or alert latency; see the [Prometheus alert rule documentation](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for the role of `for`. ### Service identity diff --git a/crates/obs/src/metrics/collectors/cluster_drive.rs b/crates/obs/src/metrics/collectors/cluster_drive.rs new file mode 100644 index 000000000..c2c039f43 --- /dev/null +++ b/crates/obs/src/metrics/collectors/cluster_drive.rs @@ -0,0 +1,150 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! One observer's global drive inventory. Runtime counters belong to local drives only. + +use crate::metrics::report::PrometheusMetric; +use crate::metrics::schema::MetricType; + +#[derive(Debug, Clone, Default)] +pub(crate) struct ClusterDriveStats { + pub server: String, + pub drive: String, + pub pool_index: String, + pub set_index: String, + pub drive_index: String, + pub disk_id: String, + pub runtime_state: String, + pub offline_duration_seconds: Option, + pub capacity_state: &'static str, + pub capacity_age_seconds: u64, + pub total_bytes: u64, + pub used_bytes: u64, + pub free_bytes: u64, +} + +pub(crate) fn collect_cluster_drive_metrics(stats: &[ClusterDriveStats]) -> Vec { + let mut metrics = Vec::with_capacity(stats.len() * 13); + for drive in stats { + let metric = |name, help, value| { + PrometheusMetric::new(name, MetricType::Gauge, help, value) + .with_label_owned("server", drive.server.clone()) + .with_label_owned("drive", drive.drive.clone()) + .with_label_owned("pool_index", drive.pool_index.clone()) + .with_label_owned("set_index", drive.set_index.clone()) + .with_label_owned("drive_index", drive.drive_index.clone()) + .with_label_owned("disk_id", drive.disk_id.clone()) + }; + metrics.push(metric( + "rustfs_cluster_drive_present", + "Configured drive slot in this observer's inventory", + 1.0, + )); + for state in ["online", "offline", "returning", "suspect", "unknown"] { + let observed = match drive.runtime_state.as_str() { + "online" | "offline" | "returning" | "suspect" => drive.runtime_state.as_str(), + _ => "unknown", + }; + metrics.push( + metric( + "rustfs_cluster_drive_runtime_state", + "Observed drive runtime state (one active state)", + f64::from(state == observed), + ) + .with_label("state", state), + ); + } + if let Some(seconds) = drive.offline_duration_seconds { + metrics.push(metric( + "rustfs_cluster_drive_offline_duration_seconds", + "Observed duration in seconds the drive has been offline", + seconds as f64, + )); + } + for state in ["live", "stale", "missing"] { + metrics.push( + metric( + "rustfs_cluster_drive_capacity_observation_state", + "Provenance of the observed drive capacity", + f64::from(state == drive.capacity_state), + ) + .with_label("state", state), + ); + } + metrics.push(metric( + "rustfs_cluster_drive_capacity_observation_age_seconds", + "Age in seconds of the drive capacity observation at collection", + drive.capacity_age_seconds as f64, + )); + // A missing capacity observation is unknown, not a zero-capacity drive. + if drive.capacity_state != "missing" { + metrics.push(metric( + "rustfs_cluster_drive_total_bytes", + "Observed total drive capacity in bytes", + drive.total_bytes as f64, + )); + metrics.push(metric( + "rustfs_cluster_drive_used_bytes", + "Observed used drive capacity in bytes", + drive.used_bytes as f64, + )); + metrics.push(metric( + "rustfs_cluster_drive_free_bytes", + "Observed free drive capacity in bytes", + drive.free_bytes as f64, + )); + } + } + metrics +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_slots_remain_visible_without_inventing_capacity_or_counters() { + let metrics = collect_cluster_drive_metrics(&[ClusterDriveStats { + server: "unreachable:9000".into(), + drive: "/data".into(), + pool_index: "1".into(), + set_index: "0".into(), + drive_index: "3".into(), + capacity_state: "missing", + ..Default::default() + }]); + assert!( + metrics + .iter() + .any(|metric| metric.name == "rustfs_cluster_drive_present" && metric.value == 1.0) + ); + assert!( + metrics + .iter() + .any(|metric| metric.name == "rustfs_cluster_drive_runtime_state" + && metric.value == 1.0 + && metric.labels.iter().any(|(key, value)| *key == "state" && value == "unknown")) + ); + assert!( + !metrics + .iter() + .any(|metric| metric.name.ends_with("_bytes") || metric.metric_type == MetricType::Counter) + ); + assert!( + metrics + .iter() + .all(|metric| metric.labels.iter().any(|(key, value)| *key == "pool_index" && value == "1")) + ); + } +} diff --git a/crates/obs/src/metrics/collectors/mod.rs b/crates/obs/src/metrics/collectors/mod.rs index cb4078669..5043202f4 100644 --- a/crates/obs/src/metrics/collectors/mod.rs +++ b/crates/obs/src/metrics/collectors/mod.rs @@ -17,6 +17,7 @@ pub mod bucket; pub mod bucket_replication; pub mod cluster; pub mod cluster_config; +pub(crate) mod cluster_drive; pub mod cluster_erasure_set; pub mod cluster_health; pub mod cluster_iam; diff --git a/crates/obs/src/metrics/collectors/system_drive.rs b/crates/obs/src/metrics/collectors/system_drive.rs index c38797bfa..a48186b4d 100644 --- a/crates/obs/src/metrics/collectors/system_drive.rs +++ b/crates/obs/src/metrics/collectors/system_drive.rs @@ -193,31 +193,35 @@ pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetail let mut metrics = Vec::with_capacity(metric_capacity); for stat in stats { + let first_metric = metrics.len(); let server_label = stat.stats.server.as_str(); let drive_label = stat.stats.drive.as_str(); let topology_labels = topology_labels(stat); - push_drive_metric( - &mut metrics, - &DRIVE_TOTAL_BYTES_MD, - stat.stats.total_bytes as f64, - server_label, - drive_label, - ); - push_drive_metric( - &mut metrics, - &DRIVE_USED_BYTES_MD, - stat.stats.used_bytes as f64, - server_label, - drive_label, - ); - push_drive_metric( - &mut metrics, - &DRIVE_FREE_BYTES_MD, - stat.stats.free_bytes as f64, - server_label, - drive_label, - ); + push_drive_metric(&mut metrics, &DRIVE_PRESENT_MD, 1.0, server_label, drive_label); + if stat.stats.capacity_observation_state != "missing" { + push_drive_metric( + &mut metrics, + &DRIVE_TOTAL_BYTES_MD, + stat.stats.total_bytes as f64, + server_label, + drive_label, + ); + push_drive_metric( + &mut metrics, + &DRIVE_USED_BYTES_MD, + stat.stats.used_bytes as f64, + server_label, + drive_label, + ); + push_drive_metric( + &mut metrics, + &DRIVE_FREE_BYTES_MD, + stat.stats.free_bytes as f64, + server_label, + drive_label, + ); + } push_drive_metric( &mut metrics, &DRIVE_CAPACITY_OBSERVATION_AGE_SECONDS_MD, @@ -352,6 +356,13 @@ pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetail ); } } + for metric in &mut metrics[first_metric..] { + if metric.metric_type == crate::metrics::schema::MetricType::Counter { + metric + .labels + .push((DISK_ID_LABEL, Cow::Owned(stat.disk_id.clone().unwrap_or_default()))); + } + } } metrics @@ -409,6 +420,46 @@ pub fn collect_process_disk_metrics( #[cfg(test)] mod tests { use super::*; + + #[test] + fn counters_follow_physical_disk_identity_and_unknown_slots_stay_present() { + let mut drive = DriveRuntimeDetailedStats { + disk_id: Some("old-disk".into()), + pool_index: Some("0".into()), + set_index: Some("0".into()), + drive_index: Some("1".into()), + api_calls: vec![("read_all".into(), 7)], + stats: DriveDetailedStats { + server: "node1:9000".into(), + drive: "/data".into(), + writes_total: Some(7), + ..Default::default() + }, + ..Default::default() + }; + for id in ["old-disk", "new-disk"] { + drive.disk_id = Some(id.into()); + let metrics = collect_drive_runtime_detailed_metrics(&[drive.clone()]); + let counters = metrics + .iter() + .filter(|metric| metric.metric_type == crate::metrics::schema::MetricType::Counter) + .collect::>(); + assert_eq!(counters.len(), 2); + assert!( + counters + .iter() + .all(|metric| metric.labels.iter().any(|(key, value)| *key == DISK_ID_LABEL && value == id)) + ); + } + drive.disk_id = None; + let metrics = collect_drive_runtime_detailed_metrics(&[drive]); + assert!( + metrics + .iter() + .any(|metric| metric.name == "rustfs_system_drive_present" && metric.value == 1.0) + ); + assert!(!metrics.iter().any(|metric| metric.name == "rustfs_system_drive_info")); + } use crate::metrics::report::report_metrics; use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; use std::collections::BTreeSet; @@ -474,7 +525,7 @@ mod tests { let metrics = collect_drive_runtime_detailed_metrics(&stats); report_metrics(&metrics); - assert_eq!(metrics.len(), 36); + assert_eq!(metrics.len(), 37); // Verify total bytes metric let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name(); @@ -513,10 +564,11 @@ mod tests { SET_INDEX_LABEL, DRIVE_INDEX_LABEL, API_LABEL, + DISK_ID_LABEL, ], ); - assert_metric_label_keys(&metrics, &DRIVE_WRITES_TOTAL_MD, 11.0, &[SERVER_LABEL, DRIVE_LABEL]); - assert_metric_label_keys(&metrics, &DRIVE_DELETES_TOTAL_MD, 4.0, &[SERVER_LABEL, DRIVE_LABEL]); + assert_metric_label_keys(&metrics, &DRIVE_WRITES_TOTAL_MD, 11.0, &[SERVER_LABEL, DRIVE_LABEL, DISK_ID_LABEL]); + assert_metric_label_keys(&metrics, &DRIVE_DELETES_TOTAL_MD, 4.0, &[SERVER_LABEL, DRIVE_LABEL, DISK_ID_LABEL]); } #[test] @@ -551,7 +603,7 @@ mod tests { let metrics = collect_drive_detailed_metrics(&stats); - assert_eq!(metrics.len(), 8); + assert_eq!(metrics.len(), 9); assert!( metrics .iter() diff --git a/crates/obs/src/metrics/mod.rs b/crates/obs/src/metrics/mod.rs index 009cc94f2..d854c1bdc 100644 --- a/crates/obs/src/metrics/mod.rs +++ b/crates/obs/src/metrics/mod.rs @@ -20,6 +20,7 @@ pub mod scheduler; pub mod schema; pub mod stats_collector; mod storage_api; +mod storage_snapshot; pub use collectors::*; pub use config::*; diff --git a/crates/obs/src/metrics/report.rs b/crates/obs/src/metrics/report.rs index b3a421f55..e8d2f0f45 100644 --- a/crates/obs/src/metrics/report.rs +++ b/crates/obs/src/metrics/report.rs @@ -69,7 +69,7 @@ fn describe_metric_once(name: &'static str, metric_type: MetricType, help: &Cow< } } -fn counter_value_from_f64(value: f64) -> Option { +pub(crate) fn counter_value_from_f64(value: f64) -> Option { if !value.is_finite() || value < 0.0 { return None; } diff --git a/crates/obs/src/metrics/scheduler.rs b/crates/obs/src/metrics/scheduler.rs index 8d527c8df..e82ee8e31 100644 --- a/crates/obs/src/metrics/scheduler.rs +++ b/crates/obs/src/metrics/scheduler.rs @@ -23,13 +23,13 @@ //! - Process disk I/O metrics //! - Host network I/O metrics +use crate::metrics::collectors::cluster_drive::collect_cluster_drive_metrics; use crate::metrics::collectors::{ AuditTargetRuntimeStats, AuditTargetStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationRuntimeStats, - DriveRuntimeDetailedStats, NotificationStats, NotificationTargetRuntimeStats, NotificationTargetStats, @@ -107,7 +107,6 @@ use crate::metrics::schema::bucket_replication::{ BUCKET_REPL_TARGET_SENT_COUNT_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, RESULT_L, TARGET_ARN_L, }; -use crate::metrics::schema::cluster::{CLUSTER_BUCKETS_TOTAL_MD, CLUSTER_OBJECTS_TOTAL_MD}; use crate::metrics::schema::cluster_usage::{ BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD, USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD, @@ -139,25 +138,23 @@ use crate::metrics::schema::scanner::{ RESULT_LABEL as SCANNER_RESULT_LABEL, SCANNER_ACTIVE_BUCKET_DRIVE_SCAN_AGE_SECONDS_MD, SCANNER_ACTIVE_BUCKET_DRIVE_SCANS_MD, SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD, SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD, SOURCE_LABEL as SCANNER_SOURCE_LABEL, }; -use crate::metrics::schema::system_drive::{ - API_LABEL as DRIVE_API_LABEL, DISK_ID_LABEL, DRIVE_API_CALLS_MD, DRIVE_API_LATENCY_BY_API_MD, DRIVE_DELETES_TOTAL_MD, - DRIVE_HEALING_MD, DRIVE_INDEX_LABEL, DRIVE_INFO_MD, DRIVE_LABEL, DRIVE_OFFLINE_DURATION_SECONDS_MD, DRIVE_RUNTIME_STATE_MD, - DRIVE_SCANNING_MD, DRIVE_WRITES_TOTAL_MD, POOL_INDEX_LABEL, SET_INDEX_LABEL, STATE_LABEL as DRIVE_STATE_LABEL, -}; use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL}; use crate::metrics::stats_collector::{ ProcessMetricBundle, collect_api_request_stats, collect_bucket_replication_bandwidth_stats, - collect_bucket_replication_stats_bundle, collect_bucket_stats, collect_cluster_and_health_stats, - collect_cluster_config_stats, collect_cluster_usage_metric_stats, collect_compression_cluster_stats, - collect_disk_and_system_drive_runtime_stats, collect_erasure_set_stats, collect_host_network_stats, collect_iam_stats, - collect_ilm_runtime_metric_stats, collect_internode_network_stats, collect_on_demand_migration_backfill_stats, - collect_on_demand_migration_stats, collect_process_metric_bundle_with, collect_replication_stats, - collect_scanner_runtime_metric_stats, collect_system_cpu_and_memory_stats_with, collect_tier_request_metric_stats, + collect_bucket_replication_stats_bundle, collect_bucket_stats, collect_cluster_config_stats, + collect_cluster_storage_snapshot, collect_cluster_usage_metric_stats, collect_compression_cluster_stats, + collect_disk_and_system_drive_runtime_stats, collect_host_network_stats, collect_iam_stats, collect_ilm_runtime_metric_stats, + collect_internode_network_stats, collect_on_demand_migration_backfill_stats, collect_on_demand_migration_stats, + collect_process_metric_bundle_with, collect_replication_stats, collect_scanner_runtime_metric_stats, + collect_system_cpu_and_memory_stats_with, collect_tier_request_metric_stats, }; +use crate::metrics::storage_snapshot::StorageSnapshotMetrics; use crate::node_identity::{SERVER_LABEL, current_local_node_identity}; use crate::telemetry::retire_metric_series; use futures_util::FutureExt; use rustfs_audit::audit_target_metrics; +use rustfs_config::METER_INTERVAL; +use rustfs_config::observability::ENV_OBS_METER_INTERVAL; use rustfs_io_metrics::ProcessSampler; use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics}; use rustfs_utils::get_env_opt_u64; @@ -171,7 +168,7 @@ use std::time::Duration; use sysinfo::{Networks, System}; use tokio::time::{Instant, Interval, MissedTickBehavior}; use tokio_util::sync::CancellationToken; -use tracing::warn; +use tracing::{error, warn}; const LOG_COMPONENT_OBS: &str = "obs"; const LOG_SUBSYSTEM_METRICS_RUNTIME: &str = "metrics_runtime"; @@ -322,137 +319,10 @@ type AuditLegacyTargetKey = String; type AuditTargetKey = (String, String); // (server, target_id) type NotificationLegacyTargetKey = (String, String); // (target_id, target_type) type NotificationTargetKey = (String, String, String); // (server, target_id, target_type) -type DriveTopologyKey = (String, String, String, String, String); // (server, drive, pool, set, drive_index) -type DriveBasicKey = (String, String); // (server, drive) -type DriveTopologyApiKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, api) -type DriveInfoKey = (String, String, String, String, String, String); // (server, drive, pool, set, drive_index, disk_id) type ScannerCycleBucketDriveResultKey = (String, String, String, String, String); // (server, cycle_scope, bucket, drive, result) type ScannerBucketDriveResultKey = (String, String, String, String); // (server, bucket, drive, result) type ScannerActiveBucketDriveKey = (String, String, String, String); // (server, source, bucket, drive) -fn drive_info_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { - stats.iter().filter_map(drive_info_key).collect() -} - -fn drive_basic_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { - stats - .iter() - .map(|stat| (stat.stats.server.clone(), stat.stats.drive.clone())) - .collect() -} - -fn retire_drive_basic_metric_series(key: &DriveBasicKey) -> usize { - let labels = [ - (SERVER_LABEL, Cow::Owned(key.0.clone())), - (DRIVE_LABEL, Cow::Owned(key.1.clone())), - ]; - retire_metric_series(&DRIVE_WRITES_TOTAL_MD.get_full_metric_name(), &labels) - + retire_metric_series(&DRIVE_DELETES_TOTAL_MD.get_full_metric_name(), &labels) -} - -fn drive_topology_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { - stats.iter().filter_map(drive_topology_key).collect() -} - -fn drive_topology_api_live_keys(stats: &[DriveRuntimeDetailedStats]) -> HashSet { - stats - .iter() - .filter_map(|stat| { - let topology = drive_topology_key(stat)?; - Some( - stat.api_calls - .iter() - .map(|(api, _)| api) - .chain(stat.api_latency_by_api_micros.iter().map(|(api, _)| api)) - .map(move |api| { - ( - topology.0.clone(), - topology.1.clone(), - topology.2.clone(), - topology.3.clone(), - topology.4.clone(), - api.clone(), - ) - }), - ) - }) - .flatten() - .collect() -} - -fn drive_topology_key(stat: &DriveRuntimeDetailedStats) -> Option { - Some(( - stat.stats.server.clone(), - stat.stats.drive.clone(), - stat.pool_index.as_ref()?.clone(), - stat.set_index.as_ref()?.clone(), - stat.drive_index.as_ref()?.clone(), - )) -} - -fn drive_info_key(stat: &DriveRuntimeDetailedStats) -> Option { - let disk_id = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty())?; - Some(( - stat.stats.server.clone(), - stat.stats.drive.clone(), - stat.pool_index.as_ref()?.clone(), - stat.set_index.as_ref()?.clone(), - stat.drive_index.as_ref()?.clone(), - disk_id.clone(), - )) -} - -fn retire_drive_info_metric_series(key: &DriveInfoKey) -> usize { - let labels = [ - (SERVER_LABEL, Cow::Owned(key.0.clone())), - (DRIVE_LABEL, Cow::Owned(key.1.clone())), - (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), - (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), - (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), - (DISK_ID_LABEL, Cow::Owned(key.5.clone())), - ]; - retire_metric_series(&DRIVE_INFO_MD.get_full_metric_name(), &labels) -} - -fn retire_drive_topology_metric_series(key: &DriveTopologyKey) -> usize { - let labels = [ - (SERVER_LABEL, Cow::Owned(key.0.clone())), - (DRIVE_LABEL, Cow::Owned(key.1.clone())), - (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), - (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), - (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), - ]; - let mut retired = 0; - for descriptor in [&DRIVE_HEALING_MD, &DRIVE_SCANNING_MD, &DRIVE_OFFLINE_DURATION_SECONDS_MD] { - retired += retire_metric_series(&descriptor.get_full_metric_name(), &labels); - } - for state in ["online", "offline", "returning", "suspect", "unknown"] { - let state_labels = [ - (SERVER_LABEL, Cow::Owned(key.0.clone())), - (DRIVE_LABEL, Cow::Owned(key.1.clone())), - (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), - (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), - (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), - (DRIVE_STATE_LABEL, Cow::Borrowed(state)), - ]; - retired += retire_metric_series(&DRIVE_RUNTIME_STATE_MD.get_full_metric_name(), &state_labels); - } - retired -} - -fn retire_drive_topology_api_metric_series(key: &DriveTopologyApiKey) -> usize { - let labels = [ - (SERVER_LABEL, Cow::Owned(key.0.clone())), - (DRIVE_LABEL, Cow::Owned(key.1.clone())), - (POOL_INDEX_LABEL, Cow::Owned(key.2.clone())), - (SET_INDEX_LABEL, Cow::Owned(key.3.clone())), - (DRIVE_INDEX_LABEL, Cow::Owned(key.4.clone())), - (DRIVE_API_LABEL, Cow::Owned(key.5.clone())), - ]; - retire_metric_series(&DRIVE_API_CALLS_MD.get_full_metric_name(), &labels) - + retire_metric_series(&DRIVE_API_LATENCY_BY_API_MD.get_full_metric_name(), &labels) -} - fn scanner_cycle_bucket_drive_result_live_keys(stats: &ScannerRuntimeStats) -> HashSet { stats .current_cycle_bucket_drive_results @@ -846,6 +716,21 @@ fn stagger_duration(period: Duration, numerator: u32, denominator: u32) -> Durat Duration::from_nanos(u64::try_from(staggered_nanos).unwrap_or(u64::MAX)) } +fn storage_snapshot_metrics(scope: &'static str, collection_interval: Duration) -> StorageSnapshotMetrics { + let export_interval = Duration::from_secs( + get_env_opt_u64(ENV_OBS_METER_INTERVAL) + .filter(|value| *value > 0) + .unwrap_or(METER_INTERVAL), + ); + let max_age = collection_interval.max(export_interval).saturating_mul(3); + StorageSnapshotMetrics::new( + opentelemetry::global::meter("rustfs.storage"), + scope, + current_local_node_identity(), + max_age, + ) +} + fn metrics_interval(period: Duration, initial_delay: Duration) -> Interval { let mut interval = tokio::time::interval_at(Instant::now() + initial_delay, period); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); @@ -1770,26 +1655,21 @@ pub fn init_metrics_runtime(token: CancellationToken) { let token_clone = token.clone(); tokio::spawn(async move { let mut interval = metrics_interval(cluster_interval, Duration::ZERO); - let mut objects_count_was_authoritative = false; - let mut buckets_count_was_authoritative = false; + let mut snapshot = storage_snapshot_metrics("cluster", cluster_interval); loop { tokio::select! { _ = interval.tick() => { run_metrics_collector_tick(health, MetricsCollectorTaskId::ClusterStats, "cluster_stats", async { - let (stats, cluster_health) = collect_cluster_and_health_stats().await; - if objects_count_was_authoritative && stats.objects_count.is_none() { - let labels: [(&'static str, Cow<'static, str>); 0] = []; - let _ = retire_metric_series(&CLUSTER_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels); + let collection_started = std::time::Instant::now(); + if let Some(stats) = collect_cluster_storage_snapshot().await { + let mut metrics = collect_cluster_metrics(&stats.cluster); + metrics.extend(collect_cluster_health_metrics(&stats.health)); + metrics.extend(collect_cluster_drive_metrics(&stats.drives)); + metrics.extend(collect_erasure_set_metrics(&stats.erasure_sets)); + if let Err(error) = snapshot.replace_collected(metrics, collection_started) { + error!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "cluster_stats", result = "invalid_snapshot", error = %error, "storage metrics snapshot rejected"); + } } - if buckets_count_was_authoritative && stats.buckets_count.is_none() { - let labels: [(&'static str, Cow<'static, str>); 0] = []; - let _ = retire_metric_series(&CLUSTER_BUCKETS_TOTAL_MD.get_full_metric_name(), &labels); - } - objects_count_was_authoritative = stats.objects_count.is_some(); - buckets_count_was_authoritative = stats.buckets_count.is_some(); - let mut metrics = collect_cluster_metrics(&stats); - metrics.extend(collect_cluster_health_metrics(&cluster_health)); - report_metrics(&metrics); }).await; } _ = token_clone.cancelled() => { @@ -1829,11 +1709,6 @@ pub fn init_metrics_runtime(token: CancellationToken) { metrics.extend(collect_cluster_config_metrics(&stats)); } - let erasure_sets = collect_erasure_set_stats().await; - if !erasure_sets.is_empty() { - metrics.extend(collect_erasure_set_metrics(&erasure_sets)); - } - if let Some(stats) = collect_iam_stats().await { metrics.extend(collect_iam_metrics(&stats)); } @@ -1994,63 +1869,19 @@ pub fn init_metrics_runtime(token: CancellationToken) { let token_clone = token.clone(); tokio::spawn(async move { let mut interval = metrics_interval(node_interval, Duration::ZERO); - let mut prev_drive_basic_keys: HashSet = HashSet::new(); - let mut prev_drive_info_keys: HashSet = HashSet::new(); - let mut prev_drive_topology_keys: HashSet = HashSet::new(); - let mut prev_drive_topology_api_keys: HashSet = HashSet::new(); - let mut has_seen_drive_info_snapshot = false; + let mut snapshot = storage_snapshot_metrics("local", node_interval); loop { tokio::select! { _ = interval.tick() => { run_metrics_collector_tick(health, MetricsCollectorTaskId::NodeDiskStats, "node_disk_stats", async { - let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_runtime_stats().await; - let current_drive_info_keys = drive_info_live_keys(&drive_stats); - let current_drive_basic_keys = drive_basic_live_keys(&drive_stats); - let current_drive_topology_keys = drive_topology_live_keys(&drive_stats); - let current_drive_topology_api_keys = drive_topology_api_live_keys(&drive_stats); - let retire_drive_info_keys = if has_seen_drive_info_snapshot { - prev_drive_info_keys.difference(¤t_drive_info_keys).cloned().collect::>() - } else { - Vec::new() - }; - let retire_drive_basic_keys = if has_seen_drive_info_snapshot { - prev_drive_basic_keys.difference(¤t_drive_basic_keys).cloned().collect::>() - } else { - Vec::new() - }; - let retire_drive_topology_keys = if has_seen_drive_info_snapshot { - prev_drive_topology_keys.difference(¤t_drive_topology_keys).cloned().collect::>() - } else { - Vec::new() - }; - let retire_drive_topology_api_keys = if has_seen_drive_info_snapshot { - prev_drive_topology_api_keys - .difference(¤t_drive_topology_api_keys) - .cloned() - .collect::>() - } else { - Vec::new() - }; - prev_drive_info_keys = current_drive_info_keys; - prev_drive_basic_keys = current_drive_basic_keys; - prev_drive_topology_keys = current_drive_topology_keys; - prev_drive_topology_api_keys = current_drive_topology_api_keys; - has_seen_drive_info_snapshot = true; - let mut metrics = collect_node_metrics(&disk_stats); - metrics.extend(collect_drive_runtime_detailed_metrics(&drive_stats)); - metrics.extend(collect_drive_count_metrics(&drive_counts)); - report_metrics(&metrics); - for key in retire_drive_info_keys { - let _ = retire_drive_info_metric_series(&key); - } - for key in retire_drive_basic_keys { - let _ = retire_drive_basic_metric_series(&key); - } - for key in retire_drive_topology_keys { - let _ = retire_drive_topology_metric_series(&key); - } - for key in retire_drive_topology_api_keys { - let _ = retire_drive_topology_api_metric_series(&key); + let collection_started = std::time::Instant::now(); + if let Some((disk_stats, drive_stats, drive_counts)) = collect_disk_and_system_drive_runtime_stats().await { + let mut metrics = collect_node_metrics(&disk_stats); + metrics.extend(collect_drive_runtime_detailed_metrics(&drive_stats)); + metrics.extend(collect_drive_count_metrics(&drive_counts)); + if let Err(error) = snapshot.replace_collected(metrics, collection_started) { + error!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "node_disk_stats", result = "invalid_snapshot", error = %error, "storage metrics snapshot rejected"); + } } }).await; } @@ -2777,24 +2608,6 @@ mod tests { (server.to_string(), target_id.to_string(), target_type.to_string()) } - fn drive_info_stat(disk_id: &str) -> DriveRuntimeDetailedStats { - DriveRuntimeDetailedStats { - pool_index: Some("0".to_string()), - set_index: Some("1".to_string()), - drive_index: Some("2".to_string()), - disk_id: Some(disk_id.to_string()), - runtime_state: Some("online".to_string()), - api_calls: vec![("read_all".to_string(), 1)], - api_latency_by_api_micros: vec![("write_all".to_string(), 2)], - stats: crate::metrics::DriveDetailedStats { - server: "server-a".to_string(), - drive: "/data1".to_string(), - ..Default::default() - }, - ..Default::default() - } - } - fn scanner_stats_with_last_result(bucket: &str) -> ScannerRuntimeStats { ScannerRuntimeStats { server: "server-a".to_string(), @@ -2834,69 +2647,6 @@ mod tests { } } - #[test] - fn drive_info_live_keys_detect_disk_identity_replacement() { - let previous = drive_info_live_keys(&[drive_info_stat("disk-old")]); - let current = drive_info_live_keys(&[drive_info_stat("disk-new")]); - let retired = previous.difference(¤t).cloned().collect::>(); - - assert!(current.contains(&( - "server-a".to_string(), - "/data1".to_string(), - "0".to_string(), - "1".to_string(), - "2".to_string(), - "disk-new".to_string(), - ))); - assert!(retired.contains(&( - "server-a".to_string(), - "/data1".to_string(), - "0".to_string(), - "1".to_string(), - "2".to_string(), - "disk-old".to_string(), - ))); - } - - #[test] - fn drive_topology_keys_detect_removed_drives() { - let previous = drive_topology_live_keys(&[drive_info_stat("disk-old")]); - let current = drive_topology_live_keys(&[]); - let retired = previous.difference(¤t).cloned().collect::>(); - - assert!(retired.contains(&( - "server-a".to_string(), - "/data1".to_string(), - "0".to_string(), - "1".to_string(), - "2".to_string(), - ))); - } - - #[test] - fn drive_topology_api_keys_detect_removed_drives() { - let previous = drive_topology_api_live_keys(&[drive_info_stat("disk-old")]); - let current = drive_topology_api_live_keys(&[]); - let retired = previous.difference(¤t).cloned().collect::>(); - - assert!(retired.contains(&( - "server-a".to_string(), - "/data1".to_string(), - "0".to_string(), - "1".to_string(), - "2".to_string(), - "read_all".to_string(), - ))); - assert!(retired.contains(&( - "server-a".to_string(), - "/data1".to_string(), - "0".to_string(), - "1".to_string(), - "2".to_string(), - "write_all".to_string(), - ))); - } - #[test] fn scanner_last_bucket_drive_result_keys_detect_superseded_cycle_results() { let previous = scanner_cycle_bucket_drive_result_live_keys(&scanner_stats_with_last_result("photos")); diff --git a/crates/obs/src/metrics/schema/system_drive.rs b/crates/obs/src/metrics/schema/system_drive.rs index f67449281..f139acd96 100644 --- a/crates/obs/src/metrics/schema/system_drive.rs +++ b/crates/obs/src/metrics/schema/system_drive.rs @@ -34,6 +34,17 @@ pub const STATE_LABEL: &str = "state"; /// All drive-related labels pub const ALL_DRIVE_LABELS: [&str; 2] = [SERVER_LABEL, DRIVE_LABEL]; +/// Physical disk identity separates counter resets caused by disk replacement. +pub const DRIVE_COUNTER_LABELS: [&str; 3] = [SERVER_LABEL, DRIVE_LABEL, DISK_ID_LABEL]; +pub const DRIVE_TOPOLOGY_API_COUNTER_LABELS: [&str; 7] = [ + SERVER_LABEL, + DRIVE_LABEL, + POOL_INDEX_LABEL, + SET_INDEX_LABEL, + DRIVE_INDEX_LABEL, + API_LABEL, + DISK_ID_LABEL, +]; /// Drive labels with erasure-set topology. pub const DRIVE_TOPOLOGY_LABELS: [&str; 5] = [ SERVER_LABEL, @@ -70,6 +81,15 @@ pub const DRIVE_TOPOLOGY_API_LABELS: [&str; 6] = [ API_LABEL, ]; +pub static DRIVE_PRESENT_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::Custom("present".to_string()), + "Configured local drive slot", + &ALL_DRIVE_LABELS, + subsystems::SYSTEM_DRIVE, + ) +}); + pub static DRIVE_INFO_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::Custom("info".to_string()), @@ -119,7 +139,7 @@ pub static DRIVE_API_CALLS_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::Custom("api_calls_total".to_string()), "Total drive API calls by operation", - &DRIVE_TOPOLOGY_API_LABELS, + &DRIVE_TOPOLOGY_API_COUNTER_LABELS, subsystems::SYSTEM_DRIVE, ) }); @@ -209,7 +229,7 @@ pub static DRIVE_TIMEOUT_ERRORS_MD: LazyLock = LazyLock::new(| new_counter_md( MetricName::DriveTimeoutErrorsTotal, "Total timeout errors on a drive", - &ALL_DRIVE_LABELS[..], + &DRIVE_COUNTER_LABELS[..], subsystems::SYSTEM_DRIVE, ) }); @@ -218,7 +238,7 @@ pub static DRIVE_IO_ERRORS_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::DriveIOErrorsTotal, "Total I/O errors on a drive", - &ALL_DRIVE_LABELS[..], + &DRIVE_COUNTER_LABELS[..], subsystems::SYSTEM_DRIVE, ) }); @@ -227,7 +247,7 @@ pub static DRIVE_AVAILABILITY_ERRORS_MD: LazyLock = LazyLock:: new_counter_md( MetricName::DriveAvailabilityErrorsTotal, "Total availability errors (I/O errors, timeouts) on a drive", - &ALL_DRIVE_LABELS[..], + &DRIVE_COUNTER_LABELS[..], subsystems::SYSTEM_DRIVE, ) }); @@ -263,7 +283,7 @@ pub static DRIVE_WRITES_TOTAL_MD: LazyLock = LazyLock::new(|| new_counter_md( MetricName::DriveWritesTotal, "Total successful write operations on a drive", - &ALL_DRIVE_LABELS[..], + &DRIVE_COUNTER_LABELS[..], subsystems::SYSTEM_DRIVE, ) }); @@ -272,7 +292,7 @@ pub static DRIVE_DELETES_TOTAL_MD: LazyLock = LazyLock::new(|| new_counter_md( MetricName::DriveDeletesTotal, "Total successful delete operations on a drive", - &ALL_DRIVE_LABELS[..], + &DRIVE_COUNTER_LABELS[..], subsystems::SYSTEM_DRIVE, ) }); diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index 1a2021b26..09eb63aca 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -18,6 +18,7 @@ //! RustFS internal sources (storage layer, bucket monitor, system info) //! and convert them to the Stats structs used by collectors. +use crate::metrics::collectors::cluster_drive::ClusterDriveStats; use crate::metrics::collectors::scanner::{ScannerActiveBucketDriveStats, ScannerBucketDriveResultStats, ScannerSourceWorkStats}; use crate::metrics::collectors::{ ApiRequestMetricSupport, ApiRequestStats, BucketReplicationBacklogStats, BucketReplicationBandwidthStats, @@ -487,13 +488,51 @@ pub struct ProcessMetricBundle { pub disk_write_bytes: u64, } +pub(crate) struct ClusterStorageSnapshot { + pub cluster: ClusterStats, + pub health: ClusterHealthStats, + pub drives: Vec, + pub erasure_sets: Vec, +} + +fn cluster_drive_stats_from_storage(storage: &ObsStorageInfo, local_server: &str) -> Vec { + storage + .disks + .iter() + .map(|disk| { + let (capacity_state, capacity_age_seconds) = disk_capacity_observation_state( + disk.capacity_observation_source.as_deref(), + disk.capacity_observation_age_seconds, + ); + ClusterDriveStats { + server: drive_server_label(&disk.endpoint, local_server), + drive: disk.drive_path.clone(), + pool_index: disk_topology_label(disk.pool_index).unwrap_or_default(), + set_index: disk_topology_label(disk.set_index).unwrap_or_default(), + drive_index: disk_topology_label(disk.disk_index).unwrap_or_default(), + disk_id: non_empty_disk_id(&disk.uuid).unwrap_or_default(), + runtime_state: disk.runtime_state.as_deref().unwrap_or("unknown").to_ascii_lowercase(), + offline_duration_seconds: disk.offline_duration_seconds, + capacity_state, + capacity_age_seconds, + total_bytes: disk.total_space, + used_bytes: disk.used_space, + free_bytes: disk.available_space, + } + }) + .collect() +} + /// Collect cluster and cluster-health statistics from a single storage snapshot. -pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthStats) { - let Some(store) = resolve_obs_object_store_handle() else { - return (ClusterStats::default(), ClusterHealthStats::default()); - }; +pub(crate) async fn collect_cluster_storage_snapshot() -> Option { + let store = resolve_obs_object_store_handle()?; let storage_info = StorageAdminApi::storage_info(store.as_ref()).await; + if storage_info.disks.is_empty() && storage_info.backend.drives_per_set.is_empty() { + return None; + } + let drives = cluster_drive_stats_from_storage(&storage_info, ¤t_local_node_identity()); + let erasure_sets = erasure_set_stats_from_backend(&storage_info, &storage_info.backend); let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum(); let usable_capacity = obs_total_usable_capacity_bytes(&storage_info); let free = obs_total_usable_capacity_free_bytes(&storage_info); @@ -536,8 +575,8 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS } } - ( - ClusterStats { + Some(ClusterStorageSnapshot { + cluster: ClusterStats { raw_capacity_bytes: raw_capacity, usable_capacity_bytes: usable_capacity, used_bytes: used, @@ -547,12 +586,22 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS objects_count, buckets_count, }, - ClusterHealthStats { + health: ClusterHealthStats { drives_offline_count: offline, drives_online_count: online, drives_count: storage_info.disks.len() as u64, }, - ) + drives, + erasure_sets, + }) +} + +/// Collect cluster statistics using the same observer snapshot as topology and health. +pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthStats) { + collect_cluster_storage_snapshot() + .await + .map(|snapshot| (snapshot.cluster, snapshot.health)) + .unwrap_or_default() } /// Collect cluster statistics from the storage layer. @@ -763,23 +812,31 @@ pub fn collect_system_memory_stats() -> MemoryStats { /// Collect node disk stats and drive stats from a single storage snapshot. pub async fn collect_disk_and_system_drive_stats() -> (Vec, Vec, DriveCountStats) { - let (disk_stats, drive_stats, drive_count_stats) = collect_disk_and_system_drive_runtime_stats().await; + let (disk_stats, drive_stats, drive_count_stats) = collect_disk_and_system_drive_runtime_stats().await.unwrap_or_default(); (disk_stats, drive_stats.into_iter().map(|stat| stat.stats).collect(), drive_count_stats) } pub(crate) async fn collect_disk_and_system_drive_runtime_stats() --> (Vec, Vec, DriveCountStats) { - let Some(store) = resolve_obs_object_store_handle() else { - return (Vec::new(), Vec::new(), DriveCountStats::default()); - }; +-> Option<(Vec, Vec, DriveCountStats)> { + let store = resolve_obs_object_store_handle()?; + let storage_info = StorageAdminApi::local_storage_info(store.as_ref()).await; + Some(local_drive_stats_from_storage(&storage_info, ¤t_local_node_identity())) +} - let storage_info = StorageAdminApi::storage_info(store.as_ref()).await; - let local_server = current_local_node_identity(); +fn local_drive_stats_from_storage( + storage_info: &ObsStorageInfo, + local_server: &str, +) -> (Vec, Vec, DriveCountStats) { let disk_stats = storage_info .disks .iter() + .filter(|disk| disk.local) + .filter(|disk| { + disk_capacity_observation_state(disk.capacity_observation_source.as_deref(), disk.capacity_observation_age_seconds).0 + != CAPACITY_OBSERVATION_MISSING + }) .map(|disk| DiskStats { - server: drive_server_label(&disk.endpoint, &local_server), + server: drive_server_label(&disk.endpoint, local_server), drive: disk.drive_path.clone(), total_bytes: disk.total_space, used_bytes: disk.used_space, @@ -792,6 +849,7 @@ pub(crate) async fn collect_disk_and_system_drive_runtime_stats() let drive_stats = storage_info .disks .iter() + .filter(|disk| disk.local) .map(|disk| { let is_online = disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref()); let (capacity_observation_state, capacity_observation_age_seconds) = disk_capacity_observation_state( @@ -832,7 +890,7 @@ pub(crate) async fn collect_disk_and_system_drive_runtime_stats() }) .unwrap_or_default(), stats: DriveDetailedStats { - server: drive_server_label(&disk.endpoint, &local_server), + server: drive_server_label(&disk.endpoint, local_server), drive: disk.drive_path.clone(), total_bytes: disk.total_space, used_bytes: disk.used_space, @@ -1714,6 +1772,43 @@ mod tests { info } + #[test] + fn local_details_exclude_remote_copies_but_keep_offline_configured_slots() { + let mut info = storage_info_with_one_online_disk(); + info.disks[0].local = true; + info.disks[0].endpoint = "http://owner:9000/data".into(); + info.disks[0].drive_path = "/data".into(); + info.disks[0].uuid = "disk-old".into(); + info.disks[0].capacity_observation_source = Some("live_probe".into()); + let mut remote = info.disks[0].clone(); + remote.local = false; + remote.endpoint = "http://peer:9000/data".into(); + remote.pool_index = 1; + remote.uuid = "peer-disk".into(); + info.disks.push(remote); + let (disks, local, counts) = local_drive_stats_from_storage(&info, "owner:9000"); + assert_eq!(disks.len(), 1); + assert_eq!(local.len(), 1); + assert_eq!(local[0].stats.server, "owner:9000"); + assert_eq!(local[0].disk_id.as_deref(), Some("disk-old")); + assert_eq!(counts.total_count, 1); + let global = cluster_drive_stats_from_storage(&info, "owner:9000"); + assert_eq!(global.len(), 2); + assert_eq!(global[1].server, "peer:9000"); + assert_eq!(global[1].pool_index, "1"); + // A disconnected configured local slot has no disk ID, but is not removed. + info.disks[0].uuid.clear(); + info.disks[0].state = "offline".into(); + info.disks[0].runtime_state = Some("offline".into()); + info.disks[0].capacity_observation_source = None; + let (_, local, counts) = local_drive_stats_from_storage(&info, "owner:9000"); + assert_eq!(local.len(), 1); + assert_eq!(local[0].disk_id, None); + assert_eq!(local[0].stats.capacity_observation_state, "missing"); + assert_eq!(counts.offline_count, 1); + assert_eq!(counts.total_count, 1); + } + #[test] fn bucket_usage_metrics_distinguish_unknown_from_confirmed_zero() { assert_eq!(bucket_usage_metric_values(None, "bucket"), (None, None)); diff --git a/crates/obs/src/metrics/storage_snapshot.rs b/crates/obs/src/metrics/storage_snapshot.rs new file mode 100644 index 000000000..4ba5f612a --- /dev/null +++ b/crates/obs/src/metrics/storage_snapshot.rs @@ -0,0 +1,407 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Current storage snapshots, without cumulative SDK retention of removed series. + +use super::report::{PrometheusMetric, counter_value_from_f64}; +use super::schema::MetricType; +use opentelemetry::{KeyValue, metrics::Meter}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub(crate) const COLLECTION_SCOPE: &str = "collection_scope"; +pub(crate) const OBSERVER: &str = "observer"; +pub(crate) const LAST_SUCCESS: &str = "rustfs_storage_snapshot_last_success_timestamp_seconds"; +pub(crate) const MAX_AGE: &str = "rustfs_storage_snapshot_max_age_seconds"; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum SnapshotError { + #[error("storage snapshots only support finite gauges and nonnegative counters")] + InvalidValue, + #[error("storage snapshot metric changed its instrument type")] + ChangedType, +} + +#[derive(Clone)] +struct Point { + value: f64, + counter: Option, + attributes: Vec, +} + +#[derive(Default)] +struct Snapshot { + points: HashMap>, + collected_at: Option, + timestamp_seconds: f64, + remaining_age: Duration, +} + +/// One collector owns each instance. Export callbacks only read its published memory. +pub(crate) struct StorageSnapshotMetrics { + meter: Meter, + snapshot: Arc>, + instruments: HashMap, + labels: Vec, + max_age: Duration, +} + +impl StorageSnapshotMetrics { + pub(crate) fn new(meter: Meter, scope: &'static str, observer: String, max_age: Duration) -> Self { + let snapshot = Arc::new(RwLock::new(Snapshot::default())); + let labels = vec![KeyValue::new(COLLECTION_SCOPE, scope), KeyValue::new(OBSERVER, observer)]; + let weak = Arc::downgrade(&snapshot); + let timestamp_labels = labels.clone(); + meter + .f64_observable_gauge(LAST_SUCCESS) + .with_description("Unix timestamp of the last successful storage snapshot collection") + .with_callback(move |observer| { + let Some(snapshot) = weak.upgrade() else { return }; + let Ok(snapshot) = snapshot.read() else { return }; + if snapshot.collected_at.is_some() { + observer.observe(snapshot.timestamp_seconds, ×tamp_labels); + } + }) + .build(); + let weak = Arc::downgrade(&snapshot); + let age_labels = labels.clone(); + meter + .f64_observable_gauge(MAX_AGE) + .with_description("Validity budget in seconds remaining after storage snapshot collection completed") + .with_callback(move |observer| { + let Some(snapshot) = weak.upgrade() else { return }; + let Ok(snapshot) = snapshot.read() else { return }; + if snapshot.collected_at.is_some() { + observer.observe(snapshot.remaining_age.as_secs_f64(), &age_labels); + } + }) + .build(); + Self { + meter, + snapshot, + instruments: HashMap::new(), + labels, + max_age, + } + } + + #[cfg(test)] + fn replace(&mut self, metrics: Vec) -> Result<(), SnapshotError> { + self.replace_collected(metrics, Instant::now()) + } + + pub(crate) fn replace_collected( + &mut self, + metrics: Vec, + collection_started: Instant, + ) -> Result<(), SnapshotError> { + // Validate the entire update before changing membership or publishing any values. + let mut types = self.instruments.clone(); + for metric in &metrics { + match metric.metric_type { + MetricType::Gauge if metric.value.is_finite() => (), + MetricType::Counter if counter_value_from_f64(metric.value).is_some() => (), + _ => return Err(SnapshotError::InvalidValue), + } + if types + .insert(metric.name.to_string(), metric.metric_type) + .is_some_and(|kind| kind != metric.metric_type) + { + return Err(SnapshotError::ChangedType); + } + } + + let mut points: HashMap> = HashMap::new(); + for metric in metrics { + let name = metric.name.to_string(); + if !self.instruments.contains_key(&name) { + self.register(&name, metric.metric_type, metric.help.as_ref())?; + self.instruments.insert(name.clone(), metric.metric_type); + } + let mut attributes = self.labels.clone(); + attributes.extend( + metric + .labels + .iter() + .map(|(key, value)| KeyValue::new(*key, value.to_string())), + ); + let point = Point { + value: metric.value, + counter: (metric.metric_type == MetricType::Counter) + .then(|| counter_value_from_f64(metric.value)) + .flatten(), + attributes, + }; + points.entry(name).or_default().push(point); + } + let timestamp_seconds = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs_f64(); + let mut snapshot = self.snapshot.write().unwrap_or_else(|poisoned| poisoned.into_inner()); + *snapshot = Snapshot { + points, + collected_at: Some(collection_started), + timestamp_seconds, + // The publication timestamp withdraws cached fields from the previous + // snapshot. Deduct collection time separately so a slow RPC or usage + // read cannot make an old observation fresh merely by completing. + remaining_age: self.max_age.saturating_sub(collection_started.elapsed()), + }; + Ok(()) + } + + fn register(&self, name: &str, kind: MetricType, description: &str) -> Result<(), SnapshotError> { + let weak = Arc::downgrade(&self.snapshot); + let name_owned = name.to_string(); + let max_age = self.max_age; + match kind { + MetricType::Counter => { + self.meter + .u64_observable_counter(name.to_string()) + .with_description(description.to_string()) + .with_callback(move |observer| { + let Some(snapshot) = weak.upgrade() else { return }; + let Ok(snapshot) = snapshot.read() else { return }; + if snapshot.collected_at.is_none_or(|when| when.elapsed() > max_age) { + return; + } + if let Some(points) = snapshot.points.get(&name_owned) { + for point in points { + if let Some(value) = point.counter { + observer.observe(value, &point.attributes); + } + } + } + }) + .build(); + } + MetricType::Gauge => { + self.meter + .f64_observable_gauge(name.to_string()) + .with_description(description.to_string()) + .with_callback(move |observer| { + let Some(snapshot) = weak.upgrade() else { return }; + let Ok(snapshot) = snapshot.read() else { return }; + if snapshot.collected_at.is_none_or(|when| when.elapsed() > max_age) { + return; + } + if let Some(points) = snapshot.points.get(&name_owned) { + for point in points { + observer.observe(point.value, &point.attributes); + } + } + }) + .build(); + } + MetricType::Histogram => return Err(SnapshotError::InvalidValue), + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::metrics::MeterProvider; + use opentelemetry_sdk::error::OTelSdkResult; + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ResourceMetrics}; + use opentelemetry_sdk::metrics::exporter::PushMetricExporter; + use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider, Temporality}; + use std::sync::Mutex; + + #[derive(Clone, Debug)] + struct ExportedPoint { + name: String, + value: f64, + attributes: Vec, + } + + #[derive(Clone, Debug, Default)] + struct Exporter(Arc>>); + + impl PushMetricExporter for Exporter { + async fn export(&self, metrics: &ResourceMetrics) -> OTelSdkResult { + let mut exported = self.0.lock().unwrap(); + exported.clear(); + for metric in metrics.scope_metrics().flat_map(|scope| scope.metrics()) { + match metric.data() { + AggregatedMetrics::F64(MetricData::Gauge(gauge)) => { + for point in gauge.data_points() { + exported.push(ExportedPoint { + name: metric.name().to_string(), + value: point.value(), + attributes: point.attributes().cloned().collect(), + }); + } + } + AggregatedMetrics::U64(MetricData::Sum(sum)) => { + assert!(sum.is_monotonic()); + for point in sum.data_points() { + exported.push(ExportedPoint { + name: metric.name().to_string(), + value: point.value() as f64, + attributes: point.attributes().cloned().collect(), + }); + } + } + data => panic!("unexpected storage aggregation: {data:?}"), + } + } + Ok(()) + } + + fn force_flush(&self) -> OTelSdkResult { + Ok(()) + } + fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult { + Ok(()) + } + fn temporality(&self) -> Temporality { + Temporality::Cumulative + } + } + + fn setup() -> (SdkMeterProvider, Exporter, StorageSnapshotMetrics) { + let exporter = Exporter::default(); + let reader = PeriodicReader::builder(exporter.clone()) + .with_interval(Duration::from_secs(3600)) + .build(); + let provider = SdkMeterProvider::builder().with_reader(reader).build(); + let snapshot = + StorageSnapshotMetrics::new(provider.meter("storage-test"), "local", "node1:9000".into(), Duration::from_secs(30)); + (provider, exporter, snapshot) + } + + fn drive(name: &'static str, kind: MetricType, id: &'static str, value: f64) -> PrometheusMetric { + PrometheusMetric::new(name, kind, "test", value) + .with_label("server", "node1:9000") + .with_label("drive", "/data") + .with_label("disk_id", id) + } + + fn flush(provider: &SdkMeterProvider, exporter: &Exporter) -> Vec { + // The SDK skips export entirely when no callback observed any point. + exporter.0.lock().unwrap().clear(); + provider.force_flush().unwrap(); + exporter.0.lock().unwrap().clone() + } + + #[test] + fn actual_sdk_drops_removed_gauges_counters_and_replaced_disk_identity() { + let (provider, exporter, mut snapshot) = setup(); + snapshot + .replace(vec![ + drive("test_info", MetricType::Gauge, "old", 1.0), + drive("test_calls", MetricType::Counter, "old", 7.0), + ]) + .unwrap(); + let first = flush(&provider, &exporter); + assert!(first.iter().any(|point| point.name == "test_calls" && point.value == 7.0)); + snapshot + .replace(vec![drive("test_info", MetricType::Gauge, "new", 1.0)]) + .unwrap(); + let second = flush(&provider, &exporter); + assert!(!second.iter().any(|point| point.name == "test_calls")); + let info = second.iter().filter(|point| point.name == "test_info").collect::>(); + assert_eq!(info.len(), 1); + assert!(info[0].attributes.contains(&KeyValue::new("disk_id", "new"))); + assert!(info[0].attributes.contains(&KeyValue::new(COLLECTION_SCOPE, "local"))); + assert!(info[0].attributes.contains(&KeyValue::new(OBSERVER, "node1:9000"))); + snapshot.replace(Vec::new()).unwrap(); + assert!( + flush(&provider, &exporter) + .iter() + .all(|point| point.name == LAST_SUCCESS || point.name == MAX_AGE) + ); + provider.shutdown().unwrap(); + } + + #[test] + fn actual_sdk_preserves_counter_resets_instead_of_accumulating_old_absolute_values() { + let (provider, exporter, mut snapshot) = setup(); + for value in [7.0, 2.0, 5.0] { + snapshot + .replace(vec![drive("test_calls", MetricType::Counter, "disk", value)]) + .unwrap(); + assert_eq!( + flush(&provider, &exporter) + .iter() + .find(|point| point.name == "test_calls") + .unwrap() + .value, + value + ); + } + provider.shutdown().unwrap(); + } + + #[test] + fn failed_or_stalled_collection_cannot_refresh_last_success_or_export_old_points() { + let (provider, exporter, mut snapshot) = setup(); + snapshot + .replace(vec![drive("test_health", MetricType::Gauge, "disk", 1.0)]) + .unwrap(); + let first = flush(&provider, &exporter); + let timestamp = first.iter().find(|point| point.name == LAST_SUCCESS).unwrap().value; + assert!( + snapshot + .replace(vec![drive("test_health", MetricType::Gauge, "disk", f64::NAN)]) + .is_err() + ); + assert!( + snapshot + .replace(vec![drive("test_health", MetricType::Counter, "disk", 3.0)]) + .is_err() + ); + assert!( + snapshot + .replace(vec![ + drive("new_name", MetricType::Gauge, "disk", 1.0), + drive("new_name", MetricType::Counter, "disk", 1.0), + ]) + .is_err() + ); + let second = flush(&provider, &exporter); + assert_eq!(second.iter().find(|point| point.name == LAST_SUCCESS).unwrap().value, timestamp); + assert!(second.iter().any(|point| point.name == "test_health" && point.value == 1.0)); + snapshot.snapshot.write().unwrap().collected_at = Some(Instant::now() - Duration::from_secs(31)); + let expired = flush(&provider, &exporter); + assert!(!expired.iter().any(|point| point.name == "test_health")); + assert_eq!(expired.iter().find(|point| point.name == LAST_SUCCESS).unwrap().value, timestamp); + drop(snapshot); + assert!(flush(&provider, &exporter).is_empty()); + provider.shutdown().unwrap(); + } + + #[test] + fn a_slow_successful_collection_cannot_publish_an_already_expired_observation() { + let (provider, exporter, mut snapshot) = setup(); + snapshot + .replace_collected( + vec![drive("test_health", MetricType::Gauge, "disk", 1.0)], + Instant::now() - Duration::from_secs(31), + ) + .expect("publish a completed but slow snapshot"); + let points = flush(&provider, &exporter); + assert!(!points.iter().any(|point| point.name == "test_health")); + assert_eq!( + points + .iter() + .find(|point| point.name == MAX_AGE) + .expect("validity budget") + .value, + 0.0 + ); + provider.shutdown().expect("shutdown test provider"); + } +} diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index c886fd683..278d2373e 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -90,6 +90,7 @@ s3s = { workspace = true, features = ["minio"] } hex-simd.workspace = true [dev-dependencies] +temp-env = { workspace = true, features = ["async_closure"] } tokio = { workspace = true, features = ["test-util"] } tokio-test = { workspace = true } criterion = { workspace = true, features = ["html_reports"] } diff --git a/crates/rio/src/encrypt_reader.rs b/crates/rio/src/encrypt_reader.rs index a579a52f5..2605316bf 100644 --- a/crates/rio/src/encrypt_reader.rs +++ b/crates/rio/src/encrypt_reader.rs @@ -327,6 +327,55 @@ where } } +/// Read-side switch for the pre-`1.0.0-alpha.91` nonce layout, in which a whole +/// v1 segment reused the part nonce for every block. +/// +/// On by default, because turning it off refuses to decrypt objects written +/// before that release. Block zero's derived nonce equals that base nonce, so +/// the layout also lets a frame encrypted at index zero authenticate anywhere +/// in its segment; the in-segment layout lock catches that as soon as a later +/// frame disagrees, but a stream that is nothing but repeats of frame zero has +/// no such later frame. A deployment with no pre-alpha.91 objects should set +/// this to `false` to remove that surface outright (backlog#2369 P2). +/// +// RUSTFS_COMPAT_TODO(backlog-2369-legacy-nonce-fallback): Remove after the +// minimum supported direct-upgrade release and after migration tooling has +// rewritten every pre-alpha.91 encrypted object. +pub const ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK: &str = "RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK"; +const DEFAULT_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK: bool = true; + +fn legacy_nonce_fallback_enabled() -> bool { + #[cfg(test)] + { + rustfs_utils::get_env_bool( + ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, + DEFAULT_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, + ) + } + #[cfg(not(test))] + { + static CACHED: std::sync::OnceLock = std::sync::OnceLock::new(); + *CACHED.get_or_init(|| { + rustfs_utils::get_env_bool( + ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, + DEFAULT_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, + ) + }) + } +} + +/// The nonce layout selected while decoding a legacy v1 segment. +/// +/// A historical writer used one of these layouts consistently for every +/// block in a segment. Once a non-zero block identifies that layout, accepting +/// another layout would let an attacker replay a block encrypted at index zero. +#[derive(Clone, Copy, PartialEq, Eq)] +enum V1NonceLayout { + Current, + LegacyBlock, + ReusedPart, +} + pin_project! { /// A reader wrapper that decrypts data on the fly using AES-256-GCM. /// This is a demonstration. For production, use a secure and audited crypto library. @@ -358,6 +407,8 @@ pin_project! { segment_frames: usize, stream_saw_v2: bool, segments_completed: usize, + v1_nonce_layout: Option, + legacy_nonce_fallback: bool, } } @@ -391,6 +442,8 @@ where segment_frames: 0, stream_saw_v2: false, segments_completed: 0, + v1_nonce_layout: None, + legacy_nonce_fallback: legacy_nonce_fallback_enabled(), } } @@ -441,6 +494,8 @@ where segment_frames: 0, stream_saw_v2: false, segments_completed: 0, + v1_nonce_layout: None, + legacy_nonce_fallback: legacy_nonce_fallback_enabled(), } } } @@ -547,6 +602,7 @@ where *this.segment_frame_version = None; *this.saw_final_frame = false; *this.segment_frames = 0; + *this.v1_nonce_layout = None; if *this.multipart_mode { let next_part = if *this.current_part_index + 1 < this.multipart_parts.len() { @@ -696,26 +752,46 @@ where *this.base_nonce }; let legacy_block_nonce = derive_block_nonce(&legacy_part_nonce, *this.block_index); - match this.cipher.decrypt(&nonce, ciphertext) { - Ok(plaintext) => plaintext, - Err(primary_err) => { - let legacy_nonce = - Nonce::try_from(legacy_block_nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?; - - match this.cipher.decrypt(&legacy_nonce, ciphertext) { - Ok(plaintext) => plaintext, - Err(_) => { - // Accept previously written streams that reused the part nonce - // for every block inside a segment. - let legacy_part_nonce = Nonce::try_from(legacy_part_nonce.as_slice()) - .map_err(|_| Error::other("invalid nonce length"))?; - this.cipher - .decrypt(&legacy_part_nonce, ciphertext) - .map_err(|_| Error::other(format!("decrypt error: {primary_err}")))? - } + let legacy_part_nonce = + Nonce::try_from(legacy_part_nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?; + let legacy_block_nonce = + Nonce::try_from(legacy_block_nonce.as_slice()).map_err(|_| Error::other("invalid nonce length"))?; + let layouts = [ + (V1NonceLayout::Current, &nonce), + (V1NonceLayout::LegacyBlock, &legacy_block_nonce), + (V1NonceLayout::ReusedPart, &legacy_part_nonce), + ]; + let selected = if *this.block_index == 0 { None } else { *this.v1_nonce_layout }; + let mut plaintext = None; + let mut last_error = None; + for (layout, candidate_nonce) in layouts { + if selected.is_some_and(|expected| expected != layout) { + continue; + } + if layout == V1NonceLayout::ReusedPart && !*this.legacy_nonce_fallback { + continue; + } + match this.cipher.decrypt(candidate_nonce, ciphertext) { + Ok(value) => { + plaintext = Some((value, layout)); + break; } + Err(error) => last_error = Some(error), } } + let (plaintext, layout) = plaintext.ok_or_else(|| { + Error::new( + std::io::ErrorKind::InvalidData, + format!( + "decrypt error: {}", + last_error.map_or_else(|| "nonce layout rejected".to_string(), |error| error.to_string()) + ), + ) + })?; + if *this.block_index > 0 && this.v1_nonce_layout.is_none() { + *this.v1_nonce_layout = Some(layout); + } + plaintext }; if *this.current_frame_type == FRAME_TYPE_V2_FINAL { *this.saw_final_frame = true; @@ -1003,6 +1079,154 @@ mod tests { assert_eq!(&decrypted, data); } + /// Encrypts `block_count` full v1 blocks, then overwrites frame one with a + /// verbatim copy of frame zero. Every frame is the same length, so the + /// stream keeps its original size and the forgery is invisible to any + /// length check. + async fn v1_stream_with_frame_zero_replayed_at_index_one(key: [u8; 32], nonce: [u8; 12], block_count: usize) -> Vec { + assert!(block_count >= 2, "a replay needs at least two frames"); + let mut data = Vec::with_capacity(ENCRYPTION_BLOCK_SIZE * block_count); + for index in 0..block_count { + data.extend(std::iter::repeat_n(0xA1u8.wrapping_add(index as u8 * 17), ENCRYPTION_BLOCK_SIZE)); + } + + let mut encrypt_reader = EncryptReader::new(Cursor::new(data), key, nonce); + let mut encrypted = Vec::new(); + encrypt_reader.read_to_end(&mut encrypted).await.expect("encrypt v1 frames"); + + // Header layout: [type][len:24][crc:32]; `len` counts the payload plus + // its own 4-byte CRC field, so the frame occupies 8 + (len - 4) bytes. + let declared_len = (encrypted[1] as usize) | ((encrypted[2] as usize) << 8) | ((encrypted[3] as usize) << 16); + let frame_len = 8 + declared_len - 4; + let replayed_first = encrypted[..frame_len].to_vec(); + encrypted[frame_len..frame_len * 2].copy_from_slice(&replayed_first); + encrypted + } + + #[tokio::test] + async fn decrypt_reader_rejects_a_replayed_first_v1_frame() { + let key = [0x11; 32]; + let nonce = [0x22; 12]; + let encrypted = v1_stream_with_frame_zero_replayed_at_index_one(key, nonce, 3).await; + + let mut decrypt_reader = DecryptReader::new(Cursor::new(encrypted), key, nonce); + let error = decrypt_reader + .read_to_end(&mut Vec::new()) + .await + .expect_err("a repeated index-zero frame must not authenticate at index one"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + } + + /// The in-segment layout lock must not cost compatibility: every legacy v1 + /// shape the fallback chain exists for still decrypts under the default. + #[tokio::test] + async fn legacy_v1_streams_still_decrypt_under_the_default_fallback() { + temp_env::async_with_vars([(ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, None::<&str>)], async { + assert!(legacy_nonce_fallback_enabled(), "the legacy nonce fallback must stay on by default"); + + let mut key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::rng().fill_bytes(&mut key); + rand::rng().fill_bytes(&mut nonce); + let mut data = vec![0u8; ENCRYPTION_BLOCK_SIZE * 3 + 17]; + rand::rng().fill(&mut data[..]); + + // Modern single-part stream. + let mut encrypted = Vec::new(); + EncryptReader::new(Cursor::new(data.clone()), key, nonce) + .read_to_end(&mut encrypted) + .await + .expect("modern v1 stream should encrypt"); + let mut decrypted = Vec::new(); + DecryptReader::new(Cursor::new(encrypted), key, nonce) + .read_to_end(&mut decrypted) + .await + .expect("modern v1 stream should decrypt"); + assert_eq!(decrypted, data); + + // Pre-alpha.91 stream that reused the part nonce for every block. + let legacy = encrypt_with_legacy_nonce_reuse(&data, key, nonce); + let mut decrypted = Vec::new(); + DecryptReader::new(Cursor::new(legacy), key, nonce) + .read_to_end(&mut decrypted) + .await + .expect("a reused-nonce legacy stream should still decrypt"); + assert_eq!(decrypted, data); + }) + .await; + } + + /// The residual after the layout lock: a stream that is nothing but repeats + /// of frame zero has no later frame to disagree with the reused-part + /// layout, so only turning the fallback off rejects it. + #[tokio::test] + async fn a_two_frame_replay_is_closed_only_by_disabling_the_legacy_fallback() { + let key = [0x33; 32]; + let nonce = [0x44; 12]; + let encrypted = v1_stream_with_frame_zero_replayed_at_index_one(key, nonce, 2).await; + + temp_env::async_with_vars([(ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, None::<&str>)], async { + let mut forged = Vec::new(); + DecryptReader::new(Cursor::new(encrypted.clone()), key, nonce) + .read_to_end(&mut forged) + .await + .expect("with the fallback on this forgery is still accepted"); + assert_eq!(forged.len(), ENCRYPTION_BLOCK_SIZE * 2); + assert_eq!( + &forged[..ENCRYPTION_BLOCK_SIZE], + &forged[ENCRYPTION_BLOCK_SIZE..], + "the accepted forgery is frame zero's plaintext twice over" + ); + }) + .await; + + temp_env::async_with_vars([(ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, Some("false"))], async { + let error = DecryptReader::new(Cursor::new(encrypted.clone()), key, nonce) + .read_to_end(&mut Vec::new()) + .await + .expect_err("with the fallback off the replayed frame must not authenticate"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + }) + .await; + } + + /// Turning the fallback off removes exactly the third layout: modern v1 + /// streams keep decrypting, pre-alpha.91 reused-nonce streams stop. + #[tokio::test] + async fn disabling_the_legacy_nonce_fallback_refuses_only_reused_part_nonces() { + let mut key = [0u8; 32]; + let mut nonce = [0u8; 12]; + rand::rng().fill_bytes(&mut key); + rand::rng().fill_bytes(&mut nonce); + let mut data = vec![0u8; ENCRYPTION_BLOCK_SIZE * 3 + 17]; + rand::rng().fill(&mut data[..]); + + let mut modern = Vec::new(); + EncryptReader::new(Cursor::new(data.clone()), key, nonce) + .read_to_end(&mut modern) + .await + .expect("modern v1 stream should encrypt"); + let legacy = encrypt_with_legacy_nonce_reuse(&data, key, nonce); + + temp_env::async_with_vars([(ENV_RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK, Some("false"))], async { + assert!(!legacy_nonce_fallback_enabled(), "the switch must be observed"); + + let mut decrypted = Vec::new(); + DecryptReader::new(Cursor::new(modern), key, nonce) + .read_to_end(&mut decrypted) + .await + .expect("modern v1 streams must keep decrypting with the fallback off"); + assert_eq!(decrypted, data); + + let error = DecryptReader::new(Cursor::new(legacy), key, nonce) + .read_to_end(&mut Vec::new()) + .await + .expect_err("the third layout must be gone when the fallback is off"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + }) + .await; + } + #[tokio::test] async fn test_decrypt_reader_only() { // Encrypt some data first diff --git a/crates/s3select-api/src/csv_input.rs b/crates/s3select-api/src/csv_input.rs new file mode 100644 index 000000000..b63d82d84 --- /dev/null +++ b/crates/s3select-api/src/csv_input.rs @@ -0,0 +1,432 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use datafusion::object_store::{Error, Result}; +use futures::{Stream, StreamExt, stream::BoxStream}; +use transform_stream::AsyncTryStream; + +use crate::SelectError; + +/// Arrow accepts byte-sized CSV controls. Unicode quotes need streaming normalization. +pub fn csv_input_requires_normalization(quote: Option<&str>, escape: Option<&str>) -> bool { + quote.is_some_and(|quote| quote.len() > 1) || escape.is_some_and(|escape| escape.len() > 1) +} + +/// CSV syntax independent of request headers, serialization formats, or S3 DTOs. +#[derive(Default)] +pub(crate) struct CsvSyntax<'a> { + pub quote: Option<&'a str>, + pub escape: Option<&'a str>, + pub field: Option<&'a str>, + pub record: Option<&'a str>, + pub comment: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + FieldStart, + Unquoted, + Quoted, + AfterQuote, + Escaped, + Comment, +} + +/// Emits ordinary CSV with every field quoted. This avoids reserving a sentinel +/// byte that might also appear in a UTF-8 field. Only a partial control token is +/// retained between chunks; neither records nor objects are buffered. +struct CsvInputNormalizer { + quote: Vec, + escape: Vec, + field: Vec, + record: Vec, + comment: Option, + default_records: bool, + state: State, + record_start: bool, + carry: Vec, + token_size: usize, +} + +impl CsvInputNormalizer { + fn new(csv: &CsvSyntax<'_>) -> Self { + let quote = csv + .quote + .filter(|value| !value.is_empty()) + .unwrap_or("\"") + .as_bytes() + .to_vec(); + let escape = csv + .escape + .filter(|value| !value.is_empty()) + .unwrap_or("\"") + .as_bytes() + .to_vec(); + let field = csv.field.filter(|value| !value.is_empty()).unwrap_or(",").as_bytes().to_vec(); + let record = csv + .record + .filter(|value| !value.is_empty()) + .unwrap_or("\n") + .as_bytes() + .to_vec(); + let token_size = quote.len().max(escape.len()).max(field.len()).max(record.len()).max(2); + Self { + quote, + escape, + field, + record, + comment: csv.comment, + default_records: csv.record.is_none(), + state: State::FieldStart, + record_start: true, + carry: Vec::new(), + token_size, + } + } + + fn record_len(&self, bytes: &[u8]) -> usize { + if self.default_records && bytes.starts_with(b"\r\n") { + 2 + } else if self.default_records && bytes.starts_with(b"\r") { + 1 + } else if bytes.starts_with(&self.record) { + self.record.len() + } else { + 0 + } + } + + fn push_value(output: &mut Vec, bytes: &[u8]) { + for byte in bytes { + if *byte == b'"' { + output.push(b'"'); + } + output.push(*byte); + } + } + + fn convert(&mut self, chunk: &[u8], last: bool) -> std::result::Result, SelectError> { + let mut bytes = std::mem::take(&mut self.carry); + bytes.extend_from_slice(chunk); + let end = if last { + bytes.len() + } else { + bytes.len().saturating_sub(self.token_size - 1) + }; + let mut output = Vec::with_capacity(bytes.len()); + let mut pos = 0; + while pos < end { + let rest = &bytes[pos..]; + let record_len = self.record_len(rest); + let field = rest.starts_with(&self.field) && self.field.len() > record_len; + match self.state { + State::Comment => { + if record_len > 0 { + self.state = State::FieldStart; + pos += record_len; + } else { + pos += 1; + } + } + State::Escaped => { + if record_len > 0 { + return Err(SelectError::CsvParsingError); + } + Self::push_value(&mut output, &rest[..1]); + self.state = State::Quoted; + pos += 1; + } + State::Quoted if rest.starts_with(&self.quote) => { + self.state = State::AfterQuote; + pos += self.quote.len(); + } + State::Quoted if rest.starts_with(&self.escape) => { + self.state = State::Escaped; + pos += self.escape.len(); + } + State::Quoted => { + if record_len > 0 { + return Err(SelectError::CsvParsingError); + } + Self::push_value(&mut output, &rest[..1]); + pos += 1; + } + State::AfterQuote if rest.starts_with(&self.quote) => { + Self::push_value(&mut output, &self.quote); + self.state = State::Quoted; + pos += self.quote.len(); + } + State::FieldStart if self.record_start && self.comment == Some(rest[0]) => { + self.state = State::Comment; + pos += 1; + } + State::FieldStart if rest.starts_with(&self.quote) => { + output.push(b'"'); + self.state = State::Quoted; + self.record_start = false; + pos += self.quote.len(); + } + _ if field || record_len > 0 => { + if self.state == State::FieldStart { + if field || !self.record_start { + output.extend_from_slice(b"\"\""); + } + } else { + output.push(b'"'); + } + output.push(if field { b',' } else { b'\n' }); + self.state = State::FieldStart; + self.record_start = !field; + pos += if field { self.field.len() } else { record_len }; + } + _ => { + if self.state == State::FieldStart { + output.push(b'"'); + } + self.state = State::Unquoted; + self.record_start = false; + Self::push_value(&mut output, &rest[..1]); + pos += 1; + } + } + } + self.carry.extend_from_slice(&bytes[pos..]); + if last { + match self.state { + State::Quoted | State::Escaped => return Err(SelectError::CsvParsingError), + State::Unquoted | State::AfterQuote => output.push(b'"'), + State::FieldStart if !self.record_start => output.extend_from_slice(b"\"\""), + State::FieldStart | State::Comment => {} + } + } + Ok(output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn normalize_chunks(csv: &CsvSyntax<'_>, input: &[u8], chunk_size: usize) -> Vec { + let mut normalizer = CsvInputNormalizer::new(csv); + let mut output = Vec::new(); + for chunk in input.chunks(chunk_size) { + output.extend(normalizer.convert(chunk, false).expect("normalize complete CSV input")); + assert!(normalizer.carry.len() < normalizer.token_size, "only a partial token may be retained"); + } + output.extend(normalizer.convert(&[], true).expect("finish complete CSV input")); + output + } + + #[test] + fn unicode_csv_quotes_preserve_values_at_every_chunk_boundary() { + let cases = [ + ("ع", "\"", "عcol1ع,عcol2ع,عcol3ع\n", "\"col1\",\"col2\",\"col3\"\n"), + ("ع", "\"", "\"left\",tail\n", "\"\"\"left\"\"\",\"tail\"\n"), + ("ع", "\"", "عA,Bع,plain\n", "\"A,B\",\"plain\"\n"), + ("ع", "\"", "عAععBع,tail\n", "\"AعB\",\"tail\"\n"), + ("ع", "\"", "عA\"عBع,tail\n", "\"AعB\",\"tail\"\n"), + ("ع", "\\", "عA\\\"Bع,tail\n", "\"A\"\"B\",\"tail\"\n"), + ("\"", "界", "\"A界\"B\",\"C\"\n", "\"A\"\"B\",\"C\"\n"), + ("🦀", "🦀", "🦀A🦀🦀B🦀,C\n", "\"A🦀B\",\"C\"\n"), + ("ع", "\"", "a\0b,عc\0dع\n", "\"a\0b\",\"c\0d\"\n"), + ("ع", "\"", "AعB,tail\n", "\"AعB\",\"tail\"\n"), + ("ع", "\"", "عaعsuffix,tail\n", "\"asuffix\",\"tail\"\n"), + ("ع", "\"", ",\n", "\"\",\"\"\n"), + ("ع", "\"", "a,", "\"a\",\"\""), + ("ع", "\"", "عع", "\"\""), + ("ع", "\"", "\n", "\n"), + ("ع", "\"", "", ""), + ]; + for (quote, escape, input, expected) in cases { + let csv = CsvSyntax { + quote: Some(quote), + escape: Some(escape), + record: Some("\n"), + ..Default::default() + }; + for chunk_size in 1..=input.len().max(1) { + assert_eq!( + normalize_chunks(&csv, input.as_bytes(), chunk_size), + expected.as_bytes(), + "input={input:?}, chunk_size={chunk_size}" + ); + } + } + } + + #[test] + fn unicode_csv_quotes_keep_custom_delimiters_and_comments_out_of_values() { + let csv = CsvSyntax { + quote: Some("ع"), + escape: Some("\\"), + field: Some("界"), + record: Some("^Y"), + comment: Some(b'#'), + }; + let input = "#skipع界^Yعa界bع界\"literal\"^Yعline\nbreakع界end^Y"; + let expected = "\"a界b\",\"\"\"literal\"\"\"\n\"line\nbreak\",\"end\"\n"; + for chunk_size in 1..=input.len() { + assert_eq!(normalize_chunks(&csv, input.as_bytes(), chunk_size), expected.as_bytes()); + } + } + + #[test] + fn unicode_csv_quotes_reject_unterminated_fields_and_quoted_record_delimiters() { + for input in ["عunfinished", "عescape\\", "عline\nbreakع\n", "عline\\\nbreakع\n"] { + let csv = CsvSyntax { + quote: Some("ع"), + escape: Some("\\"), + ..Default::default() + }; + let mut normalizer = CsvInputNormalizer::new(&csv); + assert_eq!(normalizer.convert(input.as_bytes(), true), Err(SelectError::CsvParsingError)); + } + } + + #[test] + fn unicode_csv_quotes_preserve_omitted_syntax_defaults() { + assert!(!csv_input_requires_normalization(None, None)); + assert!(!csv_input_requires_normalization(Some("\""), Some("\\"))); + assert!(csv_input_requires_normalization(Some("ع"), None)); + assert!(csv_input_requires_normalization(None, Some("界"))); + + let quote_only = CsvSyntax { + quote: Some("ع"), + ..Default::default() + }; + assert_eq!( + normalize_chunks("e_only, "عA\"عBع,tail\r\n".as_bytes(), 1), + "\"AعB\",\"tail\"\n".as_bytes() + ); + let escape_only = CsvSyntax { + escape: Some("界"), + ..Default::default() + }; + assert_eq!( + normalize_chunks(&escape_only, "\"A界\"B\",tail\r\n".as_bytes(), 1), + b"\"A\"\"B\",\"tail\"\n" + ); + } + + #[test] + fn unicode_csv_quotes_stream_large_fields_without_retaining_records() { + let csv = CsvSyntax { + quote: Some("ع"), + ..Default::default() + }; + let mut normalizer = CsvInputNormalizer::new(&csv); + let chunk = vec![b'x'; 64 * 1024]; + let mut output_len = normalizer.convert("ع".as_bytes(), false).expect("opening quote").len(); + for _ in 0..64 { + let output = normalizer.convert(&chunk, false).expect("stream field chunk"); + assert!(output.len() >= chunk.len() - 3, "field data must be emitted before its closing quote"); + assert!(normalizer.carry.len() < 4); + output_len += output.len(); + } + output_len += normalizer.convert("ع\n".as_bytes(), true).expect("close field").len(); + assert_eq!(output_len, chunk.len() * 64 + 3); + } +} + +pub(crate) fn normalize_csv_stream(stream: S, csv: &CsvSyntax<'_>) -> BoxStream<'static, Result> +where + S: Stream> + Send + 'static, +{ + let mut normalizer = CsvInputNormalizer::new(csv); + AsyncTryStream::::new(|mut y| async move { + futures::pin_mut!(stream); + while let Some(chunk) = stream.next().await { + let converted = normalizer.convert(&chunk?, false).map_err(|source| Error::Generic { + store: "EcObjectStore", + source: Box::new(source), + })?; + if !converted.is_empty() { + y.yield_ok(Bytes::from(converted)).await; + } + } + let converted = normalizer.convert(&[], true).map_err(|source| Error::Generic { + store: "EcObjectStore", + source: Box::new(source), + })?; + if !converted.is_empty() { + y.yield_ok(Bytes::from(converted)).await; + } + Ok(()) + }) + .boxed() +} + +#[cfg(test)] +mod stream_tests { + use super::*; + use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }; + + struct DropProbe(Arc); + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + #[tokio::test] + async fn unicode_csv_quotes_drop_the_source_without_reading_ahead() { + let polls = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicBool::new(false)); + let source = + futures::stream::unfold((DropProbe(Arc::clone(&dropped)), Arc::clone(&polls)), |(guard, polls)| async move { + polls.fetch_add(1, Ordering::SeqCst); + Some((Ok(Bytes::from_static("عvalueع\n".as_bytes())), (guard, polls))) + }); + let csv = CsvSyntax { + quote: Some("ع"), + ..Default::default() + }; + let mut stream = normalize_csv_stream(source, &csv); + assert!(!stream.next().await.expect("first output").expect("valid CSV").is_empty()); + assert_eq!(polls.load(Ordering::SeqCst), 1); + drop(stream); + assert!(dropped.load(Ordering::SeqCst), "cancellation must release the source reader"); + assert_eq!(polls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn unicode_csv_quotes_preserve_source_errors_after_partial_output() { + let source = futures::stream::iter([ + Ok(Bytes::from_static("عvalueع\n".as_bytes())), + Err(Error::Generic { + store: "fixture", + source: std::io::Error::other("source read failed").into(), + }), + ]); + let csv = CsvSyntax { + quote: Some("ع"), + ..Default::default() + }; + let mut stream = normalize_csv_stream(source, &csv); + assert!(!stream.next().await.expect("partial output").expect("valid prefix").is_empty()); + let error = stream + .next() + .await + .expect("source failure must remain visible") + .expect_err("must not return a successful tail"); + assert!(error.to_string().contains("source read failed")); + assert!(stream.next().await.is_none()); + } +} diff --git a/crates/s3select-api/src/lib.rs b/crates/s3select-api/src/lib.rs index a443a07a1..cebec28a6 100644 --- a/crates/s3select-api/src/lib.rs +++ b/crates/s3select-api/src/lib.rs @@ -23,12 +23,14 @@ use datafusion::{ use std::{error::Error as StdError, fmt::Display}; use thiserror::Error; +mod csv_input; mod input_stream; mod metrics; pub mod object_store; pub mod query; pub mod server; mod storage_api; +pub use csv_input::csv_input_requires_normalization; pub use metrics::{SelectInputMetrics, SelectInputMetricsSnapshot}; pub use storage_api::SelectObjectSnapshot; diff --git a/crates/s3select-api/src/object_store.rs b/crates/s3select-api/src/object_store.rs index 499436333..c2b2240d1 100644 --- a/crates/s3select-api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -64,6 +64,7 @@ use tokio::{io::AsyncReadExt, sync::OnceCell}; use tokio_util::io::ReaderStream; use transform_stream::AsyncTryStream; +use crate::csv_input::{CsvSyntax, csv_input_requires_normalization, normalize_csv_stream}; use crate::storage_api::object_store::HTTPRangeSpec; mod json_document; @@ -345,6 +346,29 @@ impl EcObjectStore { (self.need_convert || (delimiter.len() == 2 && delimiter != NORMALIZED_RECORD_DELIMITER)).then_some(delimiter) } + fn convert_csv_stream(&self, stream: S) -> BoxStream<'static, Result> + where + S: Stream> + Send + 'static, + { + if let Some(csv) = self.input.request.input_serialization.csv.as_ref() + && csv_input_requires_normalization(csv.quote_character.as_deref(), csv.quote_escape_character.as_deref()) + { + let syntax = CsvSyntax { + quote: csv.quote_character.as_deref(), + escape: csv.quote_escape_character.as_deref(), + field: csv.field_delimiter.as_deref(), + record: csv.record_delimiter.as_deref(), + comment: csv.comments.as_ref().and_then(|comment| comment.as_bytes().first().copied()), + }; + return normalize_csv_stream(stream, &syntax); + } + convert_csv_delimiter_stream( + stream, + self.record_delimiter_for_conversion(), + self.need_convert.then(|| self.delimiter.clone()), + ) + } + fn csv_has_header(&self) -> bool { self.input .request @@ -820,7 +844,6 @@ impl ObjectStore for EcObjectStore { }); } - let record_delimiter = self.record_delimiter_for_conversion(); let needs_scan_context = options.range.is_none() && has_effective_request_range; let scan_context = if needs_scan_context { if let Some(scan_range) = self.scan_range(original_size)? { @@ -883,8 +906,7 @@ impl ObjectStore for EcObjectStore { max_processed_bytes, query_guard, )?; - let stream = - convert_csv_delimiter_stream(stream, record_delimiter, self.need_convert.then(|| self.delimiter.clone())); + let stream = self.convert_csv_stream(stream); GetResultPayload::Stream(stream) } } else if options.range.is_some() { @@ -937,8 +959,7 @@ impl ObjectStore for EcObjectStore { } else { stream }; - let stream = - convert_csv_delimiter_stream(stream, record_delimiter, self.need_convert.then(|| self.delimiter.clone())); + let stream = self.convert_csv_stream(stream); GetResultPayload::Stream(stream) } else { let stream_size = usize::try_from(original_size).map_err(|err| o_Error::Generic { @@ -948,8 +969,7 @@ impl ObjectStore for EcObjectStore { let stream = bytes_stream(ReaderStream::with_capacity(reader.stream, SELECT_DEFAULT_READ_BUFFER_SIZE), stream_size); if meter_input { let stream = meter_uncompressed_input_stream(stream, Arc::clone(&self.input_metrics)); - let stream = - convert_csv_delimiter_stream(stream, record_delimiter, self.need_convert.then(|| self.delimiter.clone())); + let stream = self.convert_csv_stream(stream); GetResultPayload::Stream(stream) } else { GetResultPayload::Stream(stream.boxed()) @@ -2866,6 +2886,88 @@ mod test { assert_eq!(input_metrics.snapshot().bytes_processed, 2); } + #[tokio::test] + async fn unicode_csv_quotes_preserve_raw_offsets_and_metrics() { + const BUCKET: &str = "s3select-unicode-csv-stream"; + const HEADER: &str = "عnameع,عkindع\n"; + const SKIP: &str = "عskipع,عzeroع\n"; + const ROW: &str = "عA,Bع,عAععBع\n"; + let data = format!("{HEADER}{SKIP}{ROW}"); + let env = crate::storage_api::select_test_ecstore_env().await; + env.make_bucket(BUCKET, false).await; + for (object, compression, range_offset) in [ + ("plain.csv", None, None), + ("range.csv", None, Some(0)), + ("range-mid-character.csv", None, Some(1)), + ("gzip.csv", Some(CompressionFormat::Gzip), None), + ("bzip.csv", Some(CompressionFormat::Bzip2), None), + ] { + let bytes = match compression { + Some(format) => encode_compressed_fixture(format, data.as_bytes()).await, + None => data.as_bytes().to_vec(), + }; + let raw_size = bytes.len(); + let mut reader = SelectPutObjReader::from_vec(bytes); + env.ecstore + .put_object(BUCKET, object, &mut reader, &Default::default()) + .await + .expect("write Unicode CSV fixture"); + let mut input = (*csv_input(BUCKET, object)).clone(); + let csv = input.request.input_serialization.csv.as_mut().expect("CSV input"); + csv.file_header_info = Some(FileHeaderInfo::from_static(FileHeaderInfo::USE)); + csv.quote_character = Some("ع".to_owned()); + csv.quote_escape_character = Some("\\".to_owned()); + csv.record_delimiter = Some("\n".to_owned()); + input.request.input_serialization.compression_type = compression.map(|format| { + CompressionType::from_static(match format { + CompressionFormat::Gzip => CompressionType::GZIP, + CompressionFormat::Bzip2 => CompressionType::BZIP2, + }) + }); + let start = HEADER.len() + SKIP.len(); + if let Some(range_offset) = range_offset { + let offset = i64::try_from(start + range_offset).expect("fixture offset"); + input.request.scan_range = Some(ScanRange { + start: Some(offset), + end: Some(offset), + }); + } + let metrics = Arc::new(SelectInputMetrics::default()); + let store = EcObjectStore::build_with_snapshot( + Arc::new(input), + Arc::new(GreedyMemoryPool::new(1024 * 1024)), + None, + Arc::clone(&metrics), + prepare_test_snapshot(BUCKET, object).await, + JsonSource::default(), + ) + .expect("snapshot store"); + let result = store + .get_opts(&Path::from(object), GetOptions::default()) + .await + .expect("open Unicode CSV stream"); + let GetResultPayload::Stream(stream) = result.payload else { panic!("CSV must remain streaming") }; + let output = stream.try_collect::>().await.expect("normalize CSV stream").concat(); + let expected = match range_offset { + Some(0) => "\"name\",\"kind\"\n\"A,B\",\"AعB\"\n", + Some(_) => "\"name\",\"kind\"\n", + None => "\"name\",\"kind\"\n\"skip\",\"zero\"\n\"A,B\",\"AعB\"\n", + }; + assert_eq!(output, expected.as_bytes(), "object={object}"); + let measured = metrics.snapshot(); + if let Some(range_offset) = range_offset { + // The range reader includes one byte of delimiter context and a + // separate header read; offsets always refer to the original CSV. + let processed = u64::try_from(ROW.len() + 1 - range_offset + HEADER.len()).expect("raw range length"); + assert_eq!(measured.bytes_scanned, processed); + assert_eq!(measured.bytes_processed, processed); + } else { + assert_eq!(measured.bytes_scanned, u64::try_from(raw_size).expect("raw length")); + assert_eq!(measured.bytes_processed, u64::try_from(data.len()).expect("decoded length")); + } + } + } + #[tokio::test] async fn compressed_object_uses_one_full_stream_and_rejects_internal_ranges() { const BUCKET: &str = "s3select-compressed-object"; diff --git a/crates/s3select-api/src/query/session.rs b/crates/s3select-api/src/query/session.rs index 7df3cf60a..124c0c85d 100644 --- a/crates/s3select-api/src/query/session.rs +++ b/crates/s3select-api/src/query/session.rs @@ -456,7 +456,12 @@ impl SessionCtxFactory { .is_some_and(|compression| compression.as_str() != CompressionType::NONE); let metered_input_requires_single_file_scan = input_metrics.is_some() && context.input.request.input_serialization.parquet.is_none(); - let config = if custom_two_byte_record_delimiter + let normalized_csv_requires_single_file_scan = + context.input.request.input_serialization.csv.as_ref().is_some_and(|csv| { + crate::csv_input_requires_normalization(csv.quote_character.as_deref(), csv.quote_escape_character.as_deref()) + }); + let config = if normalized_csv_requires_single_file_scan + || custom_two_byte_record_delimiter || scan_range_requires_single_file_scan || json_document_requires_single_file_scan || compressed_input_requires_single_file_scan @@ -906,6 +911,25 @@ mod tests { assert!(!session.inner().config().options().optimizer.repartition_file_scans); } + #[tokio::test] + async fn unicode_csv_quotes_disable_file_scan_repartition() { + let mut context = test_context(); + Arc::get_mut(&mut context.input) + .expect("unique context") + .request + .input_serialization + .csv + .as_mut() + .expect("CSV input") + .quote_character = Some("ع".to_owned()); + let session = SessionCtxFactory::new(true) + .with_target_partitions(4) + .create_session_ctx(&context) + .await + .expect("Unicode CSV session"); + assert!(!session.inner().config().options().optimizer.repartition_file_scans); + } + #[tokio::test] async fn two_byte_csv_record_delimiter_disables_file_scan_repartition() { let mut context = test_context(); diff --git a/crates/s3select-query/src/dispatcher/manager.rs b/crates/s3select-query/src/dispatcher/manager.rs index 05a00e3e5..22a733fe7 100644 --- a/crates/s3select-query/src/dispatcher/manager.rs +++ b/crates/s3select-query/src/dispatcher/manager.rs @@ -53,7 +53,7 @@ use rustfs_s3select_api::{ }, }, }; -use s3s::dto::{CompressionType, FileHeaderInfo, JSONType, SelectObjectContentInput}; +use s3s::dto::{FileHeaderInfo, JSONType, SelectObjectContentInput}; use std::sync::LazyLock; use tokio::{ sync::Semaphore, @@ -430,13 +430,6 @@ impl SimpleQueryDispatcher { let path = format!("s3://{}/{}", self.input.bucket, self.input.key); let table_path = ListingTableUrl::parse(path)?; - let compressed_input = self - .input - .request - .input_serialization - .compression_type - .as_ref() - .is_some_and(|compression| compression.as_str() != CompressionType::NONE); let (listing_options, need_rename_volume_name, need_ignore_volume_name) = if let Some(csv) = self.input.request.input_serialization.csv.as_ref() { let mut need_rename_volume_name = false; @@ -485,28 +478,27 @@ impl SimpleQueryDispatcher { if let Some(quote) = csv.quote_character.as_ref() { file_format = file_format.with_quote(quote.as_bytes().first().copied().unwrap_or_default()); } + if rustfs_s3select_api::csv_input_requires_normalization( + csv.quote_character.as_deref(), + csv.quote_escape_character.as_deref(), + ) { + file_format = file_format + .with_quote(b'"') + .with_escape(None) + .with_delimiter(b',') + .with_terminator(Some(b'\n')) + .with_comment(None) + .with_newlines_in_values(true); + } ( - ListingOptions::new(Arc::new(file_format)).with_file_extension(if compressed_input { - EXACT_OBJECT_FILE_EXTENSION - } else { - ".csv" - }), + ListingOptions::new(Arc::new(file_format)).with_file_extension(EXACT_OBJECT_FILE_EXTENSION), need_rename_volume_name, need_ignore_volume_name, ) } else if self.input.request.input_serialization.json.is_some() { let file_format = JsonFormat::default(); - let file_extension = if compressed_input { - EXACT_OBJECT_FILE_EXTENSION.to_string() - } else { - std::path::Path::new(&self.input.key) - .extension() - .and_then(|extension| extension.to_str()) - .map(|extension| format!(".{extension}")) - .unwrap_or_else(|| ".json".to_string()) - }; ( - ListingOptions::new(Arc::new(file_format)).with_file_extension(file_extension), + ListingOptions::new(Arc::new(file_format)).with_file_extension(EXACT_OBJECT_FILE_EXTENSION), false, false, ) @@ -1531,6 +1523,130 @@ mod tests { assert_eq!(error.select_error(), SelectError::InvalidDataSource); } + #[tokio::test] + async fn unicode_csv_quotes_reach_arrow_without_changing_field_values() { + let cases = [ + ("ع", "\"", ",", "\n", "عcol1ع,عcol2ع,عcol3ع\n", vec![vec!["col1", "col2", "col3"]]), + ( + "ع", + "\\", + ",", + "\n", + "\"literal\",عA\\\"Bع,عAععBع\n", + vec![vec!["\"literal\"", "A\"B", "AعB"]], + ), + ("\"", "界", ",", "\n", "\"A界\"B\",🦀\n", vec![vec!["A\"B", "🦀"]]), + ("ع", "\\", "界", "^Y", "عa界bع界عline\nbreakع^Y", vec![vec!["a界b", "line\nbreak"]]), + ]; + let env = snapshot_test_env().await; + for (index, (quote, escape, field, record, data, expected)) in cases.into_iter().enumerate() { + let mut input = test_input(); + input.bucket = format!("select-unicode-quotes-{index}"); + input.key = "records".to_owned(); + let csv = input.request.input_serialization.csv.as_mut().expect("CSV input"); + csv.file_header_info = Some(FileHeaderInfo::from_static(FileHeaderInfo::NONE)); + csv.quote_character = Some(quote.to_owned()); + csv.quote_escape_character = Some(escape.to_owned()); + csv.field_delimiter = Some(field.to_owned()); + csv.record_delimiter = Some(record.to_owned()); + env.make_bucket(&input.bucket, false).await; + env.put_object_bytes(&input.bucket, &input.key, data.as_bytes().to_vec()) + .await; + let snapshot = env.prepare_select_object_snapshot(&input.bucket, &input.key).await; + let input = Arc::new(input); + let dispatcher = production_dispatcher(Arc::clone(&input)); + let query = Query::new_with_snapshot(QueryContext { input }, "SELECT * FROM S3Object".to_owned(), snapshot); + let output = dispatcher.execute_query(&query).await.expect("execute Unicode CSV query"); + let mut stream = output.into_record_batch_stream().expect("record stream"); + let mut rows = Vec::new(); + while let Some(batch) = stream.next().await { + let batch = batch.expect("Arrow must receive valid UTF-8 fields"); + for row in 0..batch.num_rows() { + rows.push( + batch + .columns() + .iter() + .map(|column| { + column + .as_any() + .downcast_ref::() + .expect("CSV string column") + .value(row) + .to_owned() + }) + .collect::>(), + ); + } + } + assert_eq!(rows, expected, "fixture={index}"); + } + } + + #[tokio::test] + async fn select_uses_input_serialization_independently_of_object_extension() { + for (key, json) in [ + ("records", false), + ("records.bin", false), + ("records", true), + ("records.csv", true), + ] { + let mut input = test_input(); + input.key = key.to_owned(); + let data = if json { + input.request.input_serialization.csv = None; + input.request.input_serialization.json = Some(s3s::dto::JSONInput { + type_: Some(JSONType::from_static(JSONType::LINES)), + }); + b"{\"value\":\"selected\"}\n".as_slice() + } else { + b"value\nselected\n".as_slice() + }; + let input = Arc::new(input); + let optimizer = Arc::new(CascadeOptimizerBuilder::default().build()); + let dispatcher = test_dispatcher_for_input( + Arc::clone(&input), + Arc::new(Semaphore::new(1)), + Duration::from_secs(30), + Arc::new(SqlQueryExecutionFactory::new(optimizer, Arc::new(LocalScheduler {}))), + ); + let query = Query::new(QueryContext { input }, "SELECT * FROM S3Object".to_owned()); + let machine = dispatcher.build_query_state_machine(query).await.expect("build query state"); + let store_url = ObjectStoreUrl::parse("s3://test-bucket").expect("test store URL"); + let store = machine + .session + .inner() + .runtime_env() + .object_store(&store_url) + .expect("test store"); + store.put(&Path::from(key), data.into()).await.expect("write selected object"); + store + .put(&Path::from(format!("{key}.other")), b"unrelated\nwrong\n".as_slice().into()) + .await + .expect("write neighboring object"); + let plan = dispatcher + .build_logical_plan(Arc::clone(&machine)) + .await + .expect("infer schema without an extension filter") + .expect("select plan"); + let output = dispatcher + .execute_logical_plan(plan, machine) + .await + .expect("execute selected object"); + let mut stream = output.into_record_batch_stream().expect("record stream"); + let mut values = Vec::new(); + while let Some(batch) = stream.next().await { + let batch = batch.expect("selected batch"); + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("string column"); + values.extend(column.iter().map(|value| value.expect("selected value").to_owned())); + } + assert_eq!(values, ["selected"], "key={key}, json={json}"); + } + } + #[tokio::test] async fn csv_query_uses_custom_record_delimiter_across_file_partitions() { const ROW_COUNT: usize = 200_000; diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 52193e834..3634dfd35 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -28,10 +28,10 @@ use metrics::{counter, describe_counter, describe_histogram, histogram}; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME, - PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeReconciliationEntry, - SizeReconciliationScope, SizeSummary, TierAccountingProof, TierStats, UNKNOWN_TIER, UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP, - UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP, UnknownTierStats, hash_path, prefix_usage_in_cache, + DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSegmentInvalidationProof, DataUsageSnapshotSetState, + LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, + SizeReconciliationEntry, SizeReconciliationScope, SizeSummary, TierAccountingProof, TierStats, UNKNOWN_TIER, + UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP, UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP, UnknownTierStats, hash_path, prefix_usage_in_cache, }; use rustfs_heal_contracts::heal_channel::HealScanMode; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; @@ -657,6 +657,11 @@ pub struct DataUsageCacheInfo { /// structural plan remains reusable across ordinary bucket writes. #[serde(default)] pub scan_execution_digest: Option, + /// Process-epoch and generation window that produced a complete set cache + /// with all known segment invalidation producers wired. This proof is + /// additive compatibility metadata; absence keeps segment reuse disabled. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub segment_invalidation_proof: Option, /// Durable bucket incarnations captured for a complete set aggregate. /// Missing or nil entries are legacy/unproven and cannot authorize /// skipping an unselected bucket in a later scoped set scan. @@ -686,6 +691,7 @@ impl Serialize for DataUsageCacheInfo { + usize::from(self.lkg_leader_epoch.is_some()) + usize::from(self.lkg_scan_plan_digest.is_some()) + usize::from(self.scan_execution_digest.is_some()) + + usize::from(self.segment_invalidation_proof.is_some()) + usize::from(!self.scan_bucket_incarnations.is_empty()); let mut state = serializer.serialize_map(Some(field_count))?; state.serialize_entry("name", &self.name)?; @@ -746,6 +752,9 @@ impl Serialize for DataUsageCacheInfo { if let Some(scan_execution_digest) = self.scan_execution_digest { state.serialize_entry("scan_execution_digest", &scan_execution_digest)?; } + if let Some(proof) = &self.segment_invalidation_proof { + state.serialize_entry("segment_invalidation_proof", proof)?; + } if !self.scan_bucket_incarnations.is_empty() { state.serialize_entry("scan_bucket_incarnations", &self.scan_bucket_incarnations)?; } diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index e1986af06..b1057d9dd 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -1095,6 +1095,7 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() { assert!(!decoded.snapshot_complete); assert!(decoded.scan_plan_digest.is_none()); assert!(decoded.scan_execution_digest.is_none()); + assert!(decoded.segment_invalidation_proof.is_none()); assert_eq!(decoded.cache_key_format, 0); } @@ -1183,6 +1184,13 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() { snapshot_complete: true, scan_plan_digest: Some(TEST_PLAN_DIGEST), scan_execution_digest: Some(DataUsageScanPlanDigest([42; 32])), + segment_invalidation_proof: Some(DataUsageSegmentInvalidationProof { + process_epoch: "scanner-process".to_string(), + generation_start: 7, + generation_end: 9, + producer_identity_coverage_complete: true, + cold_zero_walk_oracle: true, + }), cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, ..Default::default() }, @@ -1212,6 +1220,16 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() { assert!(current.info.snapshot_complete); assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); assert_eq!(current.info.scan_execution_digest, Some(DataUsageScanPlanDigest([42; 32]))); + assert_eq!( + current.info.segment_invalidation_proof, + Some(DataUsageSegmentInvalidationProof { + process_epoch: "scanner-process".to_string(), + generation_start: 7, + generation_end: 9, + producer_identity_coverage_complete: true, + cold_zero_walk_oracle: true, + }) + ); assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT); assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3)); diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index fff6245bd..5d5e7dc51 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -90,16 +90,22 @@ pub use scanner::{ ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerRecoveryIntentAcceptResult, ScannerRecoveryIntentConflict, ScannerRecoveryIntentRecord, ScannerRecoveryIntentRequest, ScannerUsageStateResetResult, accept_scanner_usage_recovery_intent, - get_scanner_usage_recovery_intent, init_data_scanner, init_scanner_with_recovery, reset_scanner_cycle_recovery, - reset_scanner_usage_state_for_full_rebuild, run_scanner_usage_recovery_intent, scanner_cycle_recovery_status, - scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_recovery_actor_sha256, scanner_topology_digest, + get_scanner_usage_recovery_intent, init_data_scanner, init_scanner_with_recovery, register_scanner_pause_backlog_retirement, + reset_scanner_cycle_recovery, reset_scanner_usage_state_for_full_rebuild, run_scanner_usage_recovery_intent, + scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status, scanner_recovery_actor_sha256, + scanner_topology_digest, }; pub use scanner_io::{ - ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, - acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket, - record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, - scanner_dirty_usage_state, scanner_maintenance_generation, + ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver, + ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError, + ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation, + acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record, + record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers, + record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change, + replay_durable_dirty_usage_producer_record, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, + scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer, }; +pub use segment_invalidation::SegmentInvalidationProducerIdentity; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; use std::sync::atomic::{AtomicU64, Ordering}; pub use storage_api::ScannerReplicationConfig as ReplicationConfig; diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 08c0633fe..5ef7118aa 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -964,9 +964,10 @@ async fn run_scanner_usage_recovery_intents_for_startup( Ok(attempted) } -/// Start normal scanning when enabled, or one resume-only cleanup attempt. +/// Start normal scanning when enabled, or one bounded recovery attempt. /// The disabled branch returns a finite task for the startup owner to join; -/// it never enables ordinary namespace scanning or accepts a new reset intent. +/// it never enables ordinary namespace scanning while it replays durable reset +/// intents and cleanup markers. pub async fn init_scanner_with_recovery( ctx: CancellationToken, storeapi: Arc, @@ -988,6 +989,17 @@ pub async fn init_scanner_with_recovery( return None; } Some(tokio::spawn(async move { + if let Err(error) = run_scanner_usage_recovery_intents_for_startup(ctx.clone(), storeapi.clone()).await { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "recovery_intent_startup_discovery_failed", + error = %error, + "Scanner recovery intent startup discovery failed" + ); + } if let Err(error) = resume_scanner_cycle_cleanup(ctx, storeapi).await { warn!( target: "rustfs::scanner", @@ -1983,10 +1995,22 @@ where if let Some((notification_system, grants)) = remote_lease_probe.as_ref() && notification_system.validate_scanner_publication_leases(grants).await.is_err() { - // A remote restart or movement flip invalidates - // the token proof; usage_store interprets this - // as a publication barrier and performs no PUT. - return Some(ScannerCycleDeferReason::DataMovement); + let remote_lease_targets = grants + .iter() + .map(|grant| { + ( + grant.host.clone(), + grant.lease.session_id.clone(), + grant.lease.movement_generation, + ) + }) + .collect::>(); + let remote_leases_valid = grants.iter().all(|grant| grant.lease.is_valid()); + return Some(scanner_remote_publication_lease_failure_defer_reason( + &remote_lease_targets, + remote_leases_valid, + probe_scanner_activity(storeapi.as_ref(), true).await, + )); } scanner_local_publication_defer_reason(storeapi.as_ref()).await } @@ -3401,6 +3425,61 @@ fn scanner_post_lease_activity_defer_reason( } } +fn scanner_remote_publication_lease_failure_defer_reason( + remote_lease_targets: &[(String, String, u64)], + remote_leases_valid: bool, + activity_after_failure: Result, +) -> ScannerCycleDeferReason { + if !remote_leases_valid { + return ScannerCycleDeferReason::DataMovement; + } + let Ok(snapshot) = activity_after_failure else { + return ScannerCycleDeferReason::ActivityBaselineUnavailable; + }; + if !scanner_activity_allows_usage_publication(&snapshot) { + return ScannerCycleDeferReason::DataMovement; + } + if scanner_publication_lease_targets_match_activity(remote_lease_targets, &snapshot) { + ScannerCycleDeferReason::ActivityBaselineUnavailable + } else { + ScannerCycleDeferReason::DataMovement + } +} + +fn scanner_publication_lease_targets_match_activity( + remote_lease_targets: &[(String, String, u64)], + activity: &ScannerActivitySnapshot, +) -> bool { + let mut expected = BTreeMap::new(); + for (host, instance_id, movement_generation) in remote_lease_targets { + if host.is_empty() + || expected + .insert(host.as_str(), (instance_id.as_str(), *movement_generation)) + .is_some() + { + return false; + } + } + + let mut observed_remote_targets = 0usize; + for (host, node_activity) in activity { + if host == LOCAL_SCANNER_ACTIVITY_NODE { + continue; + } + observed_remote_targets = observed_remote_targets.saturating_add(1); + let Some((expected_instance_id, expected_movement_generation)) = expected.get(host.as_str()) else { + return false; + }; + if node_activity.instance_id != *expected_instance_id + || node_activity.movement_generation != *expected_movement_generation + { + return false; + } + } + + observed_remote_targets == remote_lease_targets.len() +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ScannerCyclePreCommitOutcome { RecoverCacheCycle(u64), @@ -3616,7 +3695,7 @@ pub(crate) use activity::{ pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; pub use backlog::{ ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, - scanner_pause_backlog_status, + register_scanner_pause_backlog_retirement, scanner_pause_backlog_status, }; #[cfg(test)] pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test; diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 0b5c38ab5..954515ae3 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -105,8 +105,14 @@ pub(super) fn remote_dirty_usage_acknowledgement_loss_reconciled( if !acknowledged_hosts.insert(acknowledgement.host.as_str()) { return false; } - scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host) - .is_some_and(|(instance_id, _generation, pending)| instance_id == acknowledgement.instance_id && !pending) + let Some(expected_generation) = acknowledgement.expected_dirty_usage_generation() else { + return false; + }; + scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host).is_some_and( + |(instance_id, generation, pending)| { + instance_id == acknowledgement.instance_id && generation >= expected_generation && !pending + }, + ) }) } @@ -509,6 +515,15 @@ pub(crate) enum ScannerDirtyUsageAcknowledgementKind { }, } +impl ScannerDirtyUsageAcknowledgement { + fn expected_dirty_usage_generation(&self) -> Option { + match &self.kind { + ScannerDirtyUsageAcknowledgementKind::Generation(generation) => Some(*generation), + ScannerDirtyUsageAcknowledgementKind::Scoped { entries, .. } => entries.iter().map(|entry| entry.generation).max(), + } + } +} + impl From for crate::storage_api::EcstoreScannerDirtyUsageAcknowledgement { fn from(acknowledgement: ScannerDirtyUsageAcknowledgement) -> Self { match acknowledgement.kind { diff --git a/crates/scanner/src/scanner/backlog.rs b/crates/scanner/src/scanner/backlog.rs index b80b371bf..3248a37f3 100644 --- a/crates/scanner/src/scanner/backlog.rs +++ b/crates/scanner/src/scanner/backlog.rs @@ -23,6 +23,10 @@ use super::ScannerCycleOutcome; use crate::data_usage_define::DataUsageCacheRevision; use crate::storage_api::ScannerStorage; use crate::storage_api::owner::ObjectIO as _; +use crate::storage_api::owner::{ + MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica, + register_scanner_pause_backlog_retirement_planner, +}; use crate::{BUCKET_META_PREFIX, ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerObjectOptions, SetDisks}; use futures::future::join_all; use http::HeaderMap; @@ -35,7 +39,6 @@ use tokio::io::AsyncReadExt; const SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION: u16 = 1; const SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION: u16 = 1; const SCANNER_PAUSE_BACKLOG_OBJECT: &str = ".scanner-pause-backlog.json"; -const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024; const SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS: u64 = 5 * 60; const SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS: u64 = 5 * 60; const SCANNER_CATCH_UP_WINDOW_SECONDS: u64 = 60 * 60; @@ -737,6 +740,13 @@ struct LoadedScannerPauseBacklog { } impl LoadedScannerPauseBacklog { + fn all_current_replicas_stable(&self) -> bool { + // Stability of an authoritative old cohort does not seed newly writable + // members. Every current replica must retain the same rollback state + // before a new commit can replace that cohort's proof. + self.durable && self.stable_matches_ledger && !self.replicas.is_empty() && self.healthy_replicas == self.replicas.len() + } + fn status(&self, now: u64, error: Option) -> ScannerPauseBacklogStatus { let persistence_unavailable = error.is_some(); let replica_degraded = self.durable && self.stale_or_unavailable_replicas > 0; @@ -1097,7 +1107,23 @@ fn select_scanner_pause_backlog_replicas(replicas: Vec { + committed.ledger == selected + && committed.replicas.iter().all(|id| { + replicas.iter().any(|replica| { + replica.id == *id + && matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(record) + if record.stable.as_ref() == Some(&selected) + && record.committed.as_ref() == Some(committed) + ) + }) + }) + } + None => matches!(&stable_consensus, Ok(Some(stable)) if stable == &selected), + }; let healthy_replicas = replicas .iter() .filter(|replica| { @@ -1140,6 +1166,246 @@ fn select_scanner_pause_backlog_replicas(replicas: Vec Result, String> { + let mut ids = BTreeSet::new(); + let mut decoded = Vec::with_capacity(replicas.len()); + let mut has_source = false; + for replica in replicas { + let id = ScannerPauseBacklogReplicaId { + pool_index: replica.pool_index, + set_index: replica.set_index, + }; + if !ids.insert(id) { + return Err("scanner pause backlog retirement has duplicate replica membership".to_string()); + } + let state = match replica.data.as_deref() { + Some(data) if data.len() <= MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize => { + let state = decode_scanner_pause_backlog_ledger(data); + if !matches!(state, ScannerPauseBacklogReplicaState::Valid(_)) { + return Err("scanner pause backlog retirement found an invalid or unsupported native record".to_string()); + } + has_source |= replica.pool_index == source_pool_index; + state + } + None => ScannerPauseBacklogReplicaState::Missing, + _ => return Err("scanner pause backlog retirement has an oversized replica".to_string()), + }; + decoded.push(ScannerPauseBacklogReplica { + id, + revision: None, + state, + }); + } + if !has_source { + return Err("scanner pause backlog retirement has no native source record".to_string()); + } + Ok(decoded) +} + +fn verify_scanner_pause_backlog_retirement( + source_pool_index: usize, + replicas: &[ScannerPauseBacklogRetirementReplica], +) -> Result<(), String> { + let mut decoded = decode_scanner_pause_backlog_retirement_replicas(source_pool_index, replicas)?; + if decoded.iter().any(|replica| { + replica.id.pool_index != source_pool_index && matches!(replica.state, ScannerPauseBacklogReplicaState::Missing) + }) { + return Err("scanner pause backlog retirement has a missing surviving replica".to_string()); + } + // Earlier entries may already have removed a source sibling after a + // successful handoff. It contributes no stored authority to final cleanup. + decoded.retain(|replica| !matches!(replica.state, ScannerPauseBacklogReplicaState::Missing)); + let surviving = decoded + .iter() + .filter(|replica| replica.id.pool_index != source_pool_index) + .cloned() + .collect::>(); + let selected = select_scanner_pause_backlog_replicas(surviving)?; + if !selected.durable || !selected.stable_matches_ledger { + return Err( + "scanner pause backlog retirement requires stable authority on the complete surviving membership".to_string(), + ); + } + let before = select_scanner_pause_backlog_replicas(decoded)?; + if !before.durable || before.ledger != selected.ledger { + return Err("scanner pause backlog retirement would change the native ledger authority".to_string()); + } + Ok(()) +} + +fn plan_scanner_pause_backlog_retirement( + source_pool_index: usize, + replicas: &[ScannerPauseBacklogRetirementReplica], +) -> Result, String> { + // Missing entries stay in the native selection. Omitting them could turn + // an incomplete old cohort into a fabricated stable consensus. + let decoded = decode_scanner_pause_backlog_retirement_replicas(source_pool_index, replicas)?; + let surviving = decoded + .iter() + .filter(|replica| replica.id.pool_index != source_pool_index) + .cloned() + .collect::>(); + if surviving.is_empty() { + return Err("scanner pause backlog retirement has no surviving membership".to_string()); + } + let all_ids = scanner_pause_backlog_replica_ids(&decoded); + let surviving_ids = scanner_pause_backlog_replica_ids(&surviving); + let full_commit = select_scanner_pause_backlog_commit(&decoded, &all_ids)?; + let surviving_commit = select_scanner_pause_backlog_commit(&surviving, &surviving_ids)?; + let selected = select_scanner_pause_backlog_replicas(surviving.clone()); + let full_selection = select_scanner_pause_backlog_replicas(decoded.clone()); + if full_commit.is_none() + && let (Ok(all), Ok(current)) = (&full_selection, &selected) + && !all.durable + && !current.durable + { + // A failed first claim may have left only an unacknowledged candidate. + // The native selector, including every Missing replica, must establish + // the empty bootstrap state. Never promote that candidate's ledger. + if decoded.iter().any(|replica| { + matches!(&replica.state, ScannerPauseBacklogReplicaState::Valid(record) + if record.committed.as_ref().is_some_and(|committed| + committed.replicas.iter().any(|id| !all_ids.contains(id)))) + }) { + return Err("scanner pause backlog bootstrap has an unread native commit member".to_string()); + } + let ledger = claim_scanner_pause_backlog_writer(&all.ledger, unix_now())?; + let committed = commit_scanner_pause_backlog_record(None, &ledger, &surviving_ids); + let stable = stable_scanner_pause_backlog_record(&ledger, committed.committed.as_ref(), &surviving_ids); + return Ok(Some(ScannerPauseBacklogRetirementPlan { + seed_record: None, + commit_record: encode_scanner_pause_backlog_record(&committed)?, + stable_record: encode_scanner_pause_backlog_record(&stable)?, + })); + } + let ledger = if let Some(committed) = &full_commit { + if surviving_commit + .as_ref() + .is_some_and(|current| current.ledger != committed.ledger) + { + return Err("scanner pause backlog retirement found a different surviving native commit".to_string()); + } + if let Ok(current) = &selected + && current.durable + && current.ledger != committed.ledger + { + // This exception is the native stabilization of the exact full + // commit these surviving members already acknowledged. An + // independent source-only proof cannot replace their stable ledger. + let acknowledged_full_commit = surviving.iter().all(|replica| { + committed.replicas.contains(&replica.id) + && matches!(&replica.state, ScannerPauseBacklogReplicaState::Valid(record) + if record.committed.as_ref() == Some(committed)) + }); + if !acknowledged_full_commit { + return Err("scanner pause backlog retirement would replace surviving stable authority".to_string()); + } + } + // Use the same native selector that validates the full commit, rather + // than inferring authority from source epoch or physical object time. + full_selection?.ledger + } else { + // An interrupted cohort switch can invalidate the old full commit + // while all survivors still have its stable rollback point. Source + // stable fields need not match, but any valid conflicting commit above + // must have been rejected before this recovery path. + let current = selected.as_ref().map_err(|err| err.clone())?; + if !current.durable { + return Err("scanner pause backlog retirement has no proven native authority".to_string()); + } + if decoded + .iter() + .filter(|replica| replica.id.pool_index == source_pool_index) + .any(|replica| { + matches!(&replica.state, ScannerPauseBacklogReplicaState::Valid(record) + if record.stable.as_ref() != Some(¤t.ledger) + && record.committed.as_ref().is_none_or(|committed| committed.ledger != current.ledger)) + }) + { + return Err("scanner pause backlog retirement has unrelated source stable authority".to_string()); + } + current.ledger.clone() + }; + let stable_on_survivors = selected + .as_ref() + .is_ok_and(|current| current.durable && current.ledger == ledger && current.stable_matches_ledger); + let current_membership_committed = surviving_commit + .as_ref() + .is_some_and(|committed| committed.ledger == ledger && committed.replicas == surviving_ids); + if stable_on_survivors && current_membership_committed { + verify_scanner_pause_backlog_retirement(source_pool_index, replicas)?; + return Ok(None); + } + + let seed_record = if stable_on_survivors { + None + } else { + let authority = full_commit + .as_ref() + .filter(|committed| committed.ledger == ledger) + .or_else(|| surviving_commit.as_ref().filter(|committed| committed.ledger == ledger)) + .ok_or_else(|| "scanner pause backlog retirement cannot seed without a native commit proof".to_string())?; + Some(encode_scanner_pause_backlog_record(&stable_scanner_pause_backlog_record( + &ledger, + Some(authority), + &surviving_ids, + ))?) + }; + let committed = commit_scanner_pause_backlog_record(Some(&ledger), &ledger, &surviving_ids); + let stable = stable_scanner_pause_backlog_record(&ledger, committed.committed.as_ref(), &surviving_ids); + Ok(Some(ScannerPauseBacklogRetirementPlan { + seed_record, + commit_record: encode_scanner_pause_backlog_record(&committed)?, + stable_record: encode_scanner_pause_backlog_record(&stable)?, + })) +} + +fn encode_scanner_pause_backlog_record(record: &ScannerPauseBacklogReplicaRecord) -> Result, String> { + let data = serde_json::to_vec(record).map_err(|err| format!("failed to encode scanner pause backlog: {err}"))?; + if data.len() > usize::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(usize::MAX) { + return Err("scanner pause backlog exceeds its size bound".to_string()); + } + Ok(data) +} + +fn stable_scanner_pause_backlog_record( + ledger: &ScannerPauseBacklogLedger, + committed: Option<&ScannerPauseBacklogCommitRecord>, + replicas: &[ScannerPauseBacklogReplicaId], +) -> ScannerPauseBacklogReplicaRecord { + let committed = committed + .cloned() + .unwrap_or_else(|| ScannerPauseBacklogCommitRecord::new(ledger.clone(), replicas.to_vec())); + ScannerPauseBacklogReplicaRecord::new(Some(ledger.clone()), Some(committed)) +} + +fn commit_scanner_pause_backlog_record( + stable: Option<&ScannerPauseBacklogLedger>, + ledger: &ScannerPauseBacklogLedger, + replicas: &[ScannerPauseBacklogReplicaId], +) -> ScannerPauseBacklogReplicaRecord { + ScannerPauseBacklogReplicaRecord::new( + stable.cloned(), + Some(ScannerPauseBacklogCommitRecord::new(ledger.clone(), replicas.to_vec())), + ) +} + +fn claim_scanner_pause_backlog_writer(ledger: &ScannerPauseBacklogLedger, now: u64) -> Result { + let mut ledger = ledger.clone(); + ledger.claim_writer(now)?; + prepare_scanner_pause_backlog_persist(&mut ledger, now)?; + Ok(ledger) +} + async fn load_scanner_pause_backlog(storeapi: Arc) -> Result where S: ScannerStorage, @@ -1160,10 +1426,7 @@ async fn write_scanner_pause_backlog_record( where S: ScannerStorage, { - let data = serde_json::to_vec(&record).map_err(|err| format!("failed to encode scanner pause backlog: {err}"))?; - if data.len() > usize::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(usize::MAX) { - return Err("scanner pause backlog exceeds its size bound".to_string()); - } + let data = encode_scanner_pause_backlog_record(&record)?; let writable = storeapi.scanner_pause_backlog_writable_set_disks().await; if writable.is_empty() { @@ -1232,13 +1495,14 @@ async fn stabilize_scanner_pause_backlog( where S: ScannerStorage, { - let committed = loaded.authoritative_commit.clone().unwrap_or_else(|| { - ScannerPauseBacklogCommitRecord::new(loaded.ledger.clone(), scanner_pause_backlog_replica_ids(&loaded.replicas)) - }); - let record = ScannerPauseBacklogReplicaRecord::new(Some(loaded.ledger.clone()), Some(committed)); + let record = stable_scanner_pause_backlog_record( + &loaded.ledger, + loaded.authoritative_commit.as_ref(), + &scanner_pause_backlog_replica_ids(&loaded.replicas), + ); write_scanner_pause_backlog_record(storeapi.clone(), loaded, record).await?; let stabilized = load_scanner_pause_backlog(storeapi).await?; - if stabilized.ledger != loaded.ledger || !stabilized.stable_matches_ledger { + if stabilized.ledger != loaded.ledger || !stabilized.all_current_replicas_stable() { return Err("scanner pause backlog failed to stabilize its last committed generation".to_string()); } Ok(stabilized) @@ -1282,13 +1546,12 @@ where } else { loaded.clone() }; - if base.durable && !base.stable_matches_ledger { + if base.durable && !base.all_current_replicas_stable() { base = stabilize_scanner_pause_backlog(storeapi.clone(), &base).await?; } let replicas = scanner_pause_backlog_replica_ids(&base.replicas); - let committed = ScannerPauseBacklogCommitRecord::new(ledger.clone(), replicas); - let record = ScannerPauseBacklogReplicaRecord::new(base.durable.then_some(base.ledger.clone()), Some(committed)); + let record = commit_scanner_pause_backlog_record(base.durable.then_some(&base.ledger), &ledger, &replicas); write_scanner_pause_backlog_record(storeapi.clone(), &base, record).await?; match load_scanner_pause_backlog(storeapi.clone()).await { @@ -1313,9 +1576,7 @@ where { pub(super) async fn claim(storeapi: Arc, now: u64) -> Result { let loaded = load_scanner_pause_backlog(storeapi.clone()).await?; - let mut ledger = loaded.ledger.clone(); - ledger.claim_writer(now)?; - prepare_scanner_pause_backlog_persist(&mut ledger, now)?; + let ledger = claim_scanner_pause_backlog_writer(&loaded.ledger, now)?; let loaded = persist_scanner_pause_backlog(storeapi.clone(), &loaded, ledger).await?; set_runtime_error(None); let controller = Self { @@ -1537,6 +1798,1031 @@ pub(super) fn scanner_pause_backlog_now() -> u64 { mod tests { use super::*; + const NATIVE_RETIREMENT_DRIVES_PER_SET: usize = 2; + + fn run_native_retirement_test(case: C) + where + C: FnOnce() -> F + Send + 'static, + F: std::future::Future + 'static, + { + std::thread::Builder::new() + .name("native-scanner-retirement".to_string()) + .stack_size(8 * rustfs_config::DEFAULT_THREAD_STACK_SIZE) + .spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("native retirement test runtime") + .block_on(async { + tokio::time::timeout(Duration::from_secs(180), case()) + .await + .expect("native retirement scenario must finish within its fixed budget"); + }); + }) + .expect("native retirement test thread") + .join() + .expect("native retirement test completed"); + } + + async fn native_retirement_store() -> (tempfile::TempDir, Arc) { + register_scanner_pause_backlog_retirement(); + let root = tempfile::tempdir().expect("native retirement fixture directory"); + let store = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root.path(), + false, + 3, + 2, + NATIVE_RETIREMENT_DRIVES_PER_SET, + false, + ) + .await; + (root, store) + } + + async fn shutdown_native_retirement_store(store: Arc) { + if let Some(token) = store.background_cancel_token() { + token.cancel(); + } + drop(store); + for _ in 0..8 { + tokio::task::yield_now().await; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + async fn native_replica_bytes(set: &SetDisks) -> (Vec, String) { + let mut reader = set + .get_object_reader( + RUSTFS_META_BUCKET, + &SCANNER_PAUSE_BACKLOG_PATH, + None, + HeaderMap::new(), + &ScannerObjectOptions::default(), + ) + .await + .expect("native replica remains readable"); + assert!(reader.object_info.version_id.is_none_or(|version| version.is_nil())); + let revision = reader.object_info.etag.clone().expect("native CAS revision"); + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes).await.expect("native replica bytes"); + (bytes, revision) + } + + async fn assert_native_source_missing(store: &ECStore, set_index: usize) { + let source = store.pools[0].disk_set[set_index] + .get_object_reader( + RUSTFS_META_BUCKET, + &SCANNER_PAUSE_BACKLOG_PATH, + None, + HeaderMap::new(), + &ScannerObjectOptions::default(), + ) + .await; + assert!(matches!( + source, + Err(EcstoreError::ObjectNotFound(_, _) | EcstoreError::VersionNotFound(_, _, _) | EcstoreError::FileNotFound) + )); + } + + async fn native_expanded_backlog_store( + old_pool_count: usize, + unstable_source: bool, + ) -> (tempfile::TempDir, Arc, ScannerPauseBacklogLedger) { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + register_scanner_pause_backlog_retirement(); + let root = tempfile::tempdir().unwrap(); + let old = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root.path(), + false, + old_pool_count, + 2, + NATIVE_RETIREMENT_DRIVES_PER_SET, + false, + ) + .await; + let fault = unstable_source + .then(|| NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&old.pools[0].disk_set[0]), "publish", 2)); + let now = unix_now(); + let mut controller = ScannerPauseBacklogController::claim(Arc::clone(&old), now) + .await + .expect("the native controller commits the actual old cohort"); + if unstable_source { + assert!(controller.loaded.requires_reload); + let (bytes, _) = native_replica_bytes(&old.pools[0].disk_set[0]).await; + let ScannerPauseBacklogReplicaState::Valid(record) = decode_scanner_pause_backlog_ledger(&bytes) else { + panic!("actual native source record"); + }; + assert!(record.stable.is_none()); + assert_eq!(record.committed.unwrap().ledger, controller.loaded.ledger); + } else { + controller.observe(observation(now + 1, true, 4)).await; + controller.observe(observation(now + 2, false, 4)).await; + let decision = controller.begin_attempt(now + 2).await; + assert!( + matches!(decision, ScannerPauseBacklogAttemptDecision::Tracked(_)), + "expected tracked native expansion setup attempt, got {decision:?}; ledger={:?}; persistence_disabled={}; runtime_error={:?}", + controller.loaded.ledger, + controller.persistence_disabled, + runtime_error() + ); + assert!(controller.loaded.ledger.has_unfinished_attempt()); + } + let original = controller.loaded.ledger.clone(); + assert!(controller.loaded.durable); + drop(controller); + drop(fault); + old.pool_meta_write_status() + .await + .expect("healthy pool metadata keeps the background recovery loop read-only"); + shutdown_native_retirement_store(old).await; + let expanded = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root.path(), + false, + 3, + 2, + NATIVE_RETIREMENT_DRIVES_PER_SET, + false, + ) + .await; + for pool in expanded.pools.iter().skip(old_pool_count) { + for set in &pool.disk_set { + let replica = read_scanner_pause_backlog_replica(Arc::clone(set)).await; + assert!(matches!(replica.state, ScannerPauseBacklogReplicaState::Missing)); + } + } + (root, expanded, original) + } + + async fn native_expanded_retirement_store( + old_pool_count: usize, + unstable_source: bool, + ) -> (tempfile::TempDir, Arc, ScannerPauseBacklogLedger) { + let (root, expanded, original) = native_expanded_backlog_store(old_pool_count, unstable_source).await; + let (source, _) = native_replica_bytes(&expanded.pools[0].disk_set[0]).await; + expanded + .prepare_scanner_pause_backlog_retirement_for_test(0, source.len() * 2) + .await + .expect("activate the expanded source cohort"); + (root, expanded, original) + } + + async fn assert_native_writer_expansion_recovers(old_pool_count: usize, failed_pool: usize, failed_write: usize) { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + let (root, store, original) = native_expanded_backlog_store(old_pool_count, false).await; + let before = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("old cohort is authoritative before expansion publication"); + assert_eq!(before.ledger, original); + assert!(before.durable && before.stable_matches_ledger); + assert_eq!(before.persistence_state, "membership_repair_pending"); + assert_eq!(before.healthy_replicas, old_pool_count * 2); + assert_eq!(before.replica_count, 6); + let current_ids = scanner_pause_backlog_replica_ids(&before.replicas); + let old_commit = before.authoritative_commit.expect("complete old cohort proof"); + let now = unix_now().saturating_add(60); + let replacement = claim_scanner_pause_backlog_writer(&original, now).expect("next writer generation"); + let new_commit = ScannerPauseBacklogCommitRecord::new(replacement.clone(), current_ids.clone()); + let failed_id = ScannerPauseBacklogReplicaId { + pool_index: failed_pool, + set_index: 0, + }; + let fault = NativeScannerPauseBacklogWriteFault::fail_before_write( + Arc::clone(&store.pools[failed_pool].disk_set[0]), + "publish", + failed_write, + ); + let result = ScannerPauseBacklogController::claim(Arc::clone(&store), now).await; + if failed_write == 3 { + let controller = result.expect("a complete new commit survives failed stabilization"); + assert!(controller.loaded.requires_reload); + assert_eq!(controller.loaded.ledger, replacement); + } else { + let error = result.err().expect("an incomplete publication must stop the writer claim"); + assert!(error.contains("injected native scanner backlog publish"), "{error}"); + } + drop(fault); + + let recovered = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("partial publication must leave a native authority or stable rollback point"); + let expected = if failed_write == 3 { &replacement } else { &original }; + assert_eq!(&recovered.ledger, expected, "a failed publication must not fabricate or reset the ledger"); + assert!(recovered.durable); + match failed_write { + 1 => assert_eq!(recovered.authoritative_commit, Some(old_commit.clone())), + 2 => assert!(recovered.authoritative_commit.is_none(), "neither partial commit is acknowledged"), + 3 => assert_eq!(recovered.authoritative_commit, Some(new_commit.clone())), + _ => panic!("only seed, commit and stabilization writes belong in this matrix"), + } + for replica in &recovered.replicas { + if failed_write == 1 && failed_pool >= old_pool_count && replica.id == failed_id { + assert!(matches!(replica.state, ScannerPauseBacklogReplicaState::Missing)); + continue; + } + let ScannerPauseBacklogReplicaState::Valid(record) = &replica.state else { + panic!("every other member must retain its native record"); + }; + let expected_stable = if failed_write == 3 && replica.id != failed_id { + &replacement + } else { + &original + }; + let expected_commit = if failed_write == 1 || (failed_write == 2 && replica.id == failed_id) { + &old_commit + } else { + &new_commit + }; + assert_eq!(record.stable.as_ref(), Some(expected_stable)); + assert_eq!(record.committed.as_ref(), Some(expected_commit)); + } + + store.pool_meta_write_status().await.expect("healthy metadata before restart"); + shutdown_native_retirement_store(store).await; + let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root.path(), + false, + 3, + 2, + NATIVE_RETIREMENT_DRIVES_PER_SET, + false, + ) + .await; + let reloaded = load_scanner_pause_backlog(Arc::clone(&restarted)) + .await + .expect("a new store must recover from disk without the failed controller"); + assert_eq!(&reloaded.ledger, expected); + let retry_now = now.saturating_add(1); + let retry_ledger = claim_scanner_pause_backlog_writer(expected, retry_now).expect("retry writer generation"); + let retried = ScannerPauseBacklogController::claim(Arc::clone(&restarted), retry_now) + .await + .expect("retry must converge the entire expanded membership"); + assert!(!retried.loaded.requires_reload); + assert_current_native_writer_ledger(&restarted, &retry_ledger).await; + assert_eq!(retry_ledger.pending_full_scan, original.pending_full_scan); + assert_eq!(retry_ledger.dirty_usage_buckets, original.dirty_usage_buckets); + assert_eq!(retry_ledger.current_attempt_serial, original.current_attempt_serial); + assert_eq!(retry_ledger.last_finished_attempt_serial, original.current_attempt_serial); + assert_eq!(retry_ledger.consecutive_failures, original.consecutive_failures + 1); + drop(retried); + shutdown_native_retirement_store(restarted).await; + } + + async fn assert_current_native_writer_ledger(store: &Arc, expected: &ScannerPauseBacklogLedger) { + let loaded = load_scanner_pause_backlog(Arc::clone(store)) + .await + .expect("read the native writer result"); + assert_eq!(&loaded.ledger, expected); + assert!(loaded.durable && loaded.stable_matches_ledger); + assert_eq!(loaded.persistence_state, "healthy"); + assert_eq!(loaded.healthy_replicas, loaded.replicas.len()); + let ids = scanner_pause_backlog_replica_ids(&loaded.replicas); + let commit = loaded.authoritative_commit.expect("complete current membership proof"); + assert_eq!(commit.replicas, ids); + for replica in loaded.replicas { + let ScannerPauseBacklogReplicaState::Valid(record) = replica.state else { + panic!("every current member must be durable"); + }; + assert_eq!(record.stable.as_ref(), Some(expected)); + assert_eq!(record.committed.as_ref(), Some(&commit)); + } + } + + #[test] + #[serial_test::serial] + fn native_writer_expansion_seed_failure_preserves_old_authority() { + run_native_retirement_test(async || { + for old_pool_count in [1, 2] { + for failed_pool in [0, 2] { + assert_native_writer_expansion_recovers(old_pool_count, failed_pool, 1).await; + } + } + }); + } + + #[test] + #[serial_test::serial] + fn native_writer_expansion_partial_commit_recovers_after_restart() { + run_native_retirement_test(async || { + for old_pool_count in [1, 2] { + for failed_pool in [0, 2] { + assert_native_writer_expansion_recovers(old_pool_count, failed_pool, 2).await; + } + } + }); + } + + #[test] + #[serial_test::serial] + fn native_writer_expansion_stabilization_failure_recovers_after_restart() { + run_native_retirement_test(async || { + for old_pool_count in [1, 2] { + for failed_pool in [0, 2] { + assert_native_writer_expansion_recovers(old_pool_count, failed_pool, 3).await; + } + } + }); + } + + #[test] + #[serial_test::serial] + fn native_writer_expansion_seeds_stale_member_before_new_commit() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + let (_root, store, original) = native_expanded_backlog_store(1, false).await; + let target = Arc::clone(&store.pools[2].disk_set[0]); + let missing = read_scanner_pause_backlog_replica(Arc::clone(&target)).await; + let stale = ScannerPauseBacklogReplicaRecord::new(Some(durable_ledger(10)), None); + Arc::clone(&store) + .save_scanner_pause_backlog_replica( + 2, + 0, + encode_scanner_pause_backlog_record(&stale).expect("stale native record"), + missing.revision.expect("missing member revision").preconditions(), + ) + .await + .expect("persist an older rejoined member without seeding it"); + let before = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("old cohort still wins"); + assert!(before.stable_matches_ledger); + assert_eq!(before.healthy_replicas, 2); + let fault = NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&target), "publish", 1); + let now = unix_now().saturating_add(60); + let error = ScannerPauseBacklogController::claim(Arc::clone(&store), now) + .await + .err() + .expect("failed stale-member seeding must reject the claim"); + assert!(error.contains("injected native scanner backlog publish"), "{error}"); + drop(fault); + let recovered = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("the old cohort proof must survive a stale-member seed failure"); + assert_eq!(recovered.ledger, original); + assert_eq!(recovered.authoritative_commit, before.authoritative_commit); + let (bytes, _) = native_replica_bytes(&target).await; + assert_eq!(bytes, encode_scanner_pause_backlog_record(&stale).expect("unchanged stale payload")); + + let expected = claim_scanner_pause_backlog_writer(&original, now).expect("retry claims the old ledger"); + ScannerPauseBacklogController::claim(Arc::clone(&store), now) + .await + .expect("retry must seed, commit and stabilize the stale member"); + assert_current_native_writer_ledger(&store, &expected).await; + drop(target); + shutdown_native_retirement_store(store).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_writer_expansion_canceled_seed_retains_fences_and_old_authority() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + use crate::storage_api::scan::NamespaceLocking as _; + + let (_root, store, original) = native_expanded_backlog_store(1, false).await; + let barrier = + NativeScannerPauseBacklogWriteFault::pause_after_write(Arc::clone(&store.pools[2].disk_set[0]), "publish"); + let now = unix_now().saturating_add(60); + let writer_store = Arc::clone(&store); + let writer = tokio::spawn(async move { ScannerPauseBacklogController::claim(writer_store, now).await }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("a seed write must reach the native persistence barrier"); + writer.abort(); + let canceled = writer.await.err().expect("the caller was canceled"); + assert!(canceled.is_cancelled()); + let pool_meta_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, "pool.bin") + .await + .expect("durable membership lock"); + assert!(pool_meta_lock.get_write_lock_quiet(Duration::from_millis(100)).await.is_err()); + barrier.release(); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let loaded = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("cancellation cannot erase the old authority"); + assert_eq!(loaded.ledger, original); + let committed = loaded.authoritative_commit.expect("seed retains the old cohort proof"); + assert_eq!(committed.replicas, vec![replica_id(0, 0), replica_id(0, 1)]); + if loaded.healthy_replicas == loaded.replica_count { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("detached native seed owners must drain without the canceled caller"); + drop(barrier); + + let expected = claim_scanner_pause_backlog_writer(&original, now).expect("retry generation"); + ScannerPauseBacklogController::claim(Arc::clone(&store), now) + .await + .expect("a fresh caller may finish the seeded membership transition"); + assert_current_native_writer_ledger(&store, &expected).await; + drop(pool_meta_lock); + shutdown_native_retirement_store(store).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_writer_expansion_rechecks_membership_after_seeding() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + let (_root, store, original) = native_expanded_backlog_store(1, false).await; + // Model a node-local writable view learning about another pool while + // the already selected seed fan-out is in flight. Durable admission + // still uses the complete pool metadata throughout the test. + let later_member = store.pool_meta.write().await.pools.pop().expect("third pool visibility"); + let before = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("initial writable view"); + assert_eq!(before.replica_count, 4); + assert_eq!(before.healthy_replicas, 2); + let barrier = + NativeScannerPauseBacklogWriteFault::pause_after_write(Arc::clone(&store.pools[1].disk_set[1]), "publish"); + let late_failure = + NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&store.pools[2].disk_set[0]), "publish", 1); + let now = unix_now().saturating_add(60); + let writer_store = Arc::clone(&store); + let writer = tokio::spawn(async move { ScannerPauseBacklogController::claim(writer_store, now).await }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("seed publication must reach the native barrier"); + store.pool_meta.write().await.pools.push(later_member); + barrier.release(); + let error = tokio::time::timeout(Duration::from_secs(30), writer) + .await + .expect("writer must finish after the seed barrier") + .expect("writer task must not panic") + .err() + .expect("a newly visible unseeded member must block the next commit"); + assert!(error.contains("failed to stabilize its last committed generation"), "{error}"); + drop(barrier); + drop(late_failure); + + let loaded = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("the old native authority must survive membership growth during seeding"); + assert_eq!(loaded.ledger, original); + assert_eq!(loaded.authoritative_commit, before.authoritative_commit); + assert_eq!(loaded.replica_count, 6); + assert_eq!(loaded.healthy_replicas, 4); + for set in &store.pools[2].disk_set { + assert!(matches!( + read_scanner_pause_backlog_replica(Arc::clone(set)).await.state, + ScannerPauseBacklogReplicaState::Missing + )); + } + + let expected = claim_scanner_pause_backlog_writer(&original, now).expect("retry writer generation"); + ScannerPauseBacklogController::claim(Arc::clone(&store), now) + .await + .expect("retry must seed the newly visible members before committing"); + assert_current_native_writer_ledger(&store, &expected).await; + shutdown_native_retirement_store(store).await; + }); + } + + async fn assert_current_native_ledger(store: &Arc, expected: &ScannerPauseBacklogLedger) { + let loaded = load_scanner_pause_backlog(Arc::clone(store)) + .await + .expect("fresh native disk selection"); + assert_eq!(&loaded.ledger, expected, "membership repair preserves every ledger field"); + assert!( + loaded.durable && loaded.stable_matches_ledger, + "unexpected loaded state: persistence_state={}, durable={}, stable_matches_ledger={}, healthy_replicas={}, stale_or_unavailable_replicas={}", + loaded.persistence_state, + loaded.durable, + loaded.stable_matches_ledger, + loaded.healthy_replicas, + loaded.stale_or_unavailable_replicas + ); + assert_eq!(loaded.healthy_replicas, 4); + let committed = loaded.authoritative_commit.expect("complete current cohort proof"); + let mut healthy_ids = loaded + .replicas + .iter() + .filter_map(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(record) + if record.stable.as_ref() == Some(expected) + && record.committed.as_ref() == Some(&committed) + ) + .then_some(replica.id) + }) + .collect::>(); + healthy_ids.sort_unstable(); + assert_eq!(committed.replicas, healthy_ids); + for id in &committed.replicas { + let set = &store.pools[id.pool_index].disk_set[id.set_index]; + let (bytes, _) = native_replica_bytes(set).await; + let ScannerPauseBacklogReplicaState::Valid(record) = decode_scanner_pause_backlog_ledger(&bytes) else { + panic!("native survivor record"); + }; + assert_eq!(record.stable.as_ref(), Some(expected)); + assert_eq!(record.committed.as_ref(), Some(&committed)); + } + } + + #[test] + #[serial_test::serial] + fn native_retirement_expansion_repairs_missing_members_without_claiming_writer() { + run_native_retirement_test(async || { + for old_pool_count in [1, 2] { + let (_root, store, original) = native_expanded_retirement_store(old_pool_count, false).await; + for set_index in 0..2 { + store.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap(); + assert_native_source_missing(&store, set_index).await; + assert_current_native_ledger(&store, &original).await; + } + assert!(original.has_unfinished_attempt()); + shutdown_native_retirement_store(store).await; + } + }); + } + + #[test] + #[serial_test::serial] + fn native_retirement_partial_native_phases_resume_from_persisted_authority() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + for phase in ["seed", "commit", "stabilize"] { + let (root, store, original) = native_expanded_retirement_store(2, true).await; + let (source, _) = native_replica_bytes(&store.pools[0].disk_set[0]).await; + let failed_pool = if phase == "seed" { 2 } else { 1 }; + let fault = NativeScannerPauseBacklogWriteFault::fail_before_write( + Arc::clone(&store.pools[failed_pool].disk_set[0]), + phase, + 1, + ); + let error = store.retire_scanner_pause_backlog_for_test(0, 0).await.unwrap_err(); + assert!( + error + .to_string() + .contains(&format!("injected native scanner backlog {phase}")), + "{error}" + ); + assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source); + let written = read_scanner_pause_backlog_replica(Arc::clone(&store.pools[2].disk_set[1])).await; + let ScannerPauseBacklogReplicaState::Valid(record) = written.state else { + panic!("a sibling must perform its actual native CAS before the phase returns an error"); + }; + assert_eq!(record.stable, Some(original.clone())); + if phase == "commit" { + let current = load_scanner_pause_backlog(Arc::clone(&store)).await.unwrap(); + assert_eq!(current.ledger, original); + assert!( + current.authoritative_commit.is_none(), + "partial old and new proofs must not be acknowledged" + ); + } + drop(fault); + store + .pool_meta_write_status() + .await + .expect("healthy pool metadata keeps the background recovery loop read-only"); + shutdown_native_retirement_store(store).await; + let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root.path(), + false, + 3, + 2, + NATIVE_RETIREMENT_DRIVES_PER_SET, + false, + ) + .await; + for set_index in 0..2 { + restarted.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap(); + assert_native_source_missing(&restarted, set_index).await; + } + assert_current_native_ledger(&restarted, &original).await; + shutdown_native_retirement_store(restarted).await; + } + }); + } + + #[test] + #[serial_test::serial] + fn native_retirement_bootstraps_an_unacknowledged_first_claim_and_retries_partial_commit() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + + let (root, store) = native_retirement_store().await; + let future_now = unix_now().saturating_add(10_000); + let fault = + NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&store.pools[1].disk_set[0]), "publish", 1); + assert!( + ScannerPauseBacklogController::claim(Arc::clone(&store), future_now) + .await + .is_err() + ); + drop(fault); + let empty = load_scanner_pause_backlog(Arc::clone(&store)).await.unwrap(); + assert!(!empty.durable); + assert_eq!(empty.ledger, ScannerPauseBacklogLedger::default()); + let (source, _) = native_replica_bytes(&store.pools[0].disk_set[0]).await; + store + .prepare_scanner_pause_backlog_retirement_for_test(0, source.len() * 2) + .await + .unwrap(); + let fault = + NativeScannerPauseBacklogWriteFault::fail_before_write(Arc::clone(&store.pools[2].disk_set[0]), "commit", 1); + let error = store.retire_scanner_pause_backlog_for_test(0, 0).await.unwrap_err(); + assert!(error.to_string().contains("injected native scanner backlog commit"), "{error}"); + assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source); + assert!(!load_scanner_pause_backlog(Arc::clone(&store)).await.unwrap().durable); + drop(fault); + store + .pool_meta_write_status() + .await + .expect("healthy pool metadata keeps the background recovery loop read-only"); + shutdown_native_retirement_store(store).await; + let restarted = super::super::tests::setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root.path(), + false, + 3, + 2, + NATIVE_RETIREMENT_DRIVES_PER_SET, + false, + ) + .await; + for set_index in 0..2 { + restarted.retire_scanner_pause_backlog_for_test(0, set_index).await.unwrap(); + assert_native_source_missing(&restarted, set_index).await; + } + let bootstrapped = load_scanner_pause_backlog(Arc::clone(&restarted)).await.unwrap(); + assert_eq!(bootstrapped.ledger.writer_epoch, 1); + assert_eq!(bootstrapped.ledger.generation, 1); + assert!(bootstrapped.ledger.last_updated_at_unix_secs < future_now); + assert_current_native_ledger(&restarted, &bootstrapped.ledger).await; + shutdown_native_retirement_store(restarted).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_retirement_canceled_waiter_keeps_fences_through_partial_native_write() { + run_native_retirement_test(async || { + use crate::storage_api::owner::NativeScannerPauseBacklogWriteFault; + use crate::storage_api::scan::NamespaceLocking as _; + + let (_root, store, original) = native_expanded_retirement_store(2, true).await; + let barrier = + NativeScannerPauseBacklogWriteFault::pause_after_write(Arc::clone(&store.pools[1].disk_set[0]), "commit"); + let caller_store = Arc::clone(&store); + let caller = tokio::spawn(async move { caller_store.retire_scanner_pause_backlog_for_test(0, 0).await }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + let pool_meta_lock = store.new_ns_lock(RUSTFS_META_BUCKET, "pool.bin").await.unwrap(); + assert!(pool_meta_lock.get_write_lock_quiet(Duration::from_millis(100)).await.is_err()); + let writer_store = Arc::clone(&store); + let mut writer = tokio::spawn(async move { + writer_store + .save_scanner_pause_backlog_replica( + 1, + 0, + b"must not replace the native record".to_vec(), + crate::storage_api::owner::HTTPPreconditions { + if_match: Some("deliberately-stale-revision".to_string()), + ..Default::default() + }, + ) + .await + }); + assert!(tokio::time::timeout(Duration::from_millis(100), &mut writer).await.is_err()); + barrier.release(); + let result = tokio::time::timeout(Duration::from_secs(30), writer).await.unwrap().unwrap(); + assert!(matches!(result, Err(EcstoreError::PreconditionFailed))); + store.retire_scanner_pause_backlog_for_test(0, 0).await.unwrap(); + assert_native_source_missing(&store, 0).await; + assert_current_native_ledger(&store, &original).await; + store.retire_scanner_pause_backlog_for_test(0, 1).await.unwrap(); + assert_native_source_missing(&store, 1).await; + drop(pool_meta_lock); + shutdown_native_retirement_store(store).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_retirement_old_cohort_survives_multiset_cleanup_and_canceled_waiter() { + run_native_retirement_test(async || { + let (_root, store) = native_retirement_store().await; + let now = unix_now(); + let seeded = ScannerPauseBacklogController::claim(Arc::clone(&store), now) + .await + .expect("native controller commits its actual record to every set"); + let original = seeded.loaded.ledger.clone(); + assert_eq!(seeded.loaded.healthy_replicas, 6); + drop(seeded); + let (source_bytes, _) = native_replica_bytes(&store.pools[0].disk_set[0]).await; + store + .prepare_scanner_pause_backlog_retirement_for_test(0, source_bytes.len() * 2) + .await + .expect("activate durable retirement with the old full-cohort record intact"); + let after_activation = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("native stable rollback"); + assert_eq!(after_activation.ledger, original); + assert!(after_activation.authoritative_commit.is_none()); + assert!(after_activation.stable_matches_ledger); + + store + .retire_scanner_pause_backlog_for_test(0, 0) + .await + .expect("retire the first source set"); + assert_native_source_missing(&store, 0).await; + store + .retire_scanner_pause_backlog_for_test(0, 0) + .await + .expect("an already removed source entry can resume"); + + let (target_bytes, target_revision) = native_replica_bytes(&store.pools[1].disk_set[0]).await; + let barrier = + crate::storage_api::owner::SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &SCANNER_PAUSE_BACKLOG_PATH); + let caller_store = Arc::clone(&store); + let caller = tokio::spawn(async move { caller_store.retire_scanner_pause_backlog_for_test(0, 1).await }); + tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused()) + .await + .expect("native retirement reaches physical source deletion"); + caller.abort(); + assert!(caller.await.expect_err("the entry waiter was canceled").is_cancelled()); + use crate::storage_api::scan::NamespaceLocking as _; + let pool_meta_lock = store.new_ns_lock(RUSTFS_META_BUCKET, "pool.bin").await.unwrap(); + assert!( + pool_meta_lock.get_write_lock_quiet(Duration::from_millis(100)).await.is_err(), + "canceling the waiter must retain the durable pool metadata snapshot fence" + ); + let writer_store = Arc::clone(&store); + let unchanged = target_bytes.clone(); + let mut writer = tokio::spawn(async move { + writer_store + .save_scanner_pause_backlog_replica( + 1, + 0, + unchanged, + crate::storage_api::owner::HTTPPreconditions { + if_match: Some(target_revision), + ..Default::default() + }, + ) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut writer).await.is_err(), + "canceling the entry waiter must not release the native snapshot fence before physical cleanup" + ); + barrier.release(); + tokio::time::timeout(Duration::from_secs(30), writer) + .await + .expect("native writer resumes within its lock budget") + .expect("native writer task") + .expect("native CAS resumes after cleanup"); + assert_native_source_missing(&store, 1).await; + store + .retire_scanner_pause_backlog_for_test(0, 1) + .await + .expect("retry after owned cleanup finished"); + let after = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("reload surviving native records"); + assert_eq!(after.ledger, original); + assert!(after.durable && after.stable_matches_ledger); + for set in store.scanner_pause_backlog_writable_set_disks().await { + assert_eq!( + native_replica_bytes(&set).await.0, + target_bytes, + "retirement never copied a stale source record" + ); + } + drop(pool_meta_lock); + drop(barrier); + shutdown_native_retirement_store(store).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_retirement_missing_or_invalid_survivor_preserves_source_until_native_repair() { + run_native_retirement_test(async || { + use crate::storage_api::owner::ObjectOperations as _; + + let (_root, store) = native_retirement_store().await; + let seeded = ScannerPauseBacklogController::claim(Arc::clone(&store), unix_now()) + .await + .expect("seed the real native replica schema"); + let old_ledger = seeded.loaded.ledger.clone(); + drop(seeded); + let (source_bytes, _) = native_replica_bytes(&store.pools[0].disk_set[0]).await; + store.pools[0].disk_set[0] + .put_object( + RUSTFS_META_BUCKET, + &SCANNER_PAUSE_BACKLOG_PATH, + &mut crate::ScannerPutObjReader::from_vec(source_bytes.clone()), + &ScannerObjectOptions { + max_parity: true, + mod_time: Some(time::OffsetDateTime::now_utc() + time::Duration::hours(1)), + ..Default::default() + }, + ) + .await + .expect("a source clock skew must not outrank the native ledger commit"); + store + .prepare_scanner_pause_backlog_retirement_for_test(0, source_bytes.len() * 2) + .await + .expect("activate the source retirement"); + let hash_set = store.pools[1].get_disks_by_key(&SCANNER_PAUSE_BACKLOG_PATH).set_index; + let missing_set = 1 - hash_set; + let target = &store.pools[1].disk_set[missing_set]; + let (target_bytes, _) = native_replica_bytes(target).await; + target + .delete_object( + RUSTFS_META_BUCKET, + &SCANNER_PAUSE_BACKLOG_PATH, + ScannerObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + ..Default::default() + }, + ) + .await + .expect("remove a non-hash-routed survivor to model replica loss"); + let missing = store + .retire_scanner_pause_backlog_for_test(0, 0) + .await + .expect_err("every surviving set is required"); + assert!( + missing + .to_string() + .contains("neither a surviving membership commit nor a stable rollback point"), + "{missing}" + ); + assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source_bytes); + assert!( + target + .get_object_reader( + RUSTFS_META_BUCKET, + &SCANNER_PAUSE_BACKLOG_PATH, + None, + HeaderMap::new(), + &ScannerObjectOptions::default() + ) + .await + .is_err(), + "retirement must not fill the missing survivor with its stale source payload" + ); + Arc::clone(&store) + .save_scanner_pause_backlog_replica( + 1, + missing_set, + target_bytes.clone(), + crate::storage_api::owner::HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + ) + .await + .expect("native CAS repairs the missing replica"); + for invalid in [b"invalid native JSON".to_vec(), br#"{"replica_schema_version":999}"#.to_vec()] { + let (_, revision) = native_replica_bytes(target).await; + Arc::clone(&store) + .save_scanner_pause_backlog_replica( + 1, + missing_set, + invalid.clone(), + crate::storage_api::owner::HTTPPreconditions { + if_match: Some(revision), + ..Default::default() + }, + ) + .await + .expect("inject a corrupt or unsupported native record through the storage write path"); + store + .retire_scanner_pause_backlog_for_test(0, 0) + .await + .expect_err("invalid native records must retain the source"); + assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source_bytes); + let (actual, revision) = native_replica_bytes(target).await; + assert_eq!(actual, invalid); + Arc::clone(&store) + .save_scanner_pause_backlog_replica( + 1, + missing_set, + target_bytes.clone(), + crate::storage_api::owner::HTTPPreconditions { + if_match: Some(revision), + ..Default::default() + }, + ) + .await + .expect("restore the native replica with CAS"); + } + let replacement = ScannerPauseBacklogController::claim(Arc::clone(&store), unix_now().saturating_add(1)) + .await + .expect("native writer advances the surviving membership"); + let new_ledger = replacement.loaded.ledger.clone(); + assert!(new_ledger.writer_epoch > old_ledger.writer_epoch); + drop(replacement); + store + .retire_scanner_pause_backlog_for_test(0, 0) + .await + .expect("the native commit supersedes the old source"); + store + .retire_scanner_pause_backlog_for_test(0, 1) + .await + .expect("the remaining source set follows the same native proof"); + let after = load_scanner_pause_backlog(Arc::clone(&store)) + .await + .expect("native restart selection after handoff"); + assert_eq!(after.ledger, new_ledger); + assert!(after.durable && after.stable_matches_ledger); + shutdown_native_retirement_store(store).await; + }); + } + + #[test] + #[serial_test::serial] + fn native_retirement_preserves_exact_target_intent_without_blocking_another_source_set() { + run_native_retirement_test(async || { + let (_root, store) = native_retirement_store().await; + let seeded = ScannerPauseBacklogController::claim(Arc::clone(&store), unix_now()) + .await + .expect("seed the real native record on every set"); + drop(seeded); + let (source_bytes, _) = native_replica_bytes(&store.pools[0].disk_set[0]).await; + store.pools[0].disk_set[1] + .put_object( + RUSTFS_META_BUCKET, + &SCANNER_PAUSE_BACKLOG_PATH, + &mut crate::ScannerPutObjReader::from_vec(source_bytes.clone()), + &ScannerObjectOptions { + max_parity: true, + mod_time: Some(time::OffsetDateTime::now_utc() + time::Duration::seconds(1)), + ..Default::default() + }, + ) + .await + .expect("the second set has the same native payload with a distinct physical revision"); + store + .prepare_scanner_pause_backlog_retirement_for_test(0, source_bytes.len() * 2) + .await + .expect("activate native retirement"); + store + .stage_scanner_pause_backlog_retirement_intent_for_test(0, 0) + .await + .expect("retain a real target intent after failure in the admitted operation"); + let before = { + let meta = store.pool_meta.read().await; + let reservation = meta.pools[0] + .decommission + .as_ref() + .unwrap() + .capacity_reservation + .as_ref() + .unwrap(); + assert!(reservation.pending_target_physical_bytes > 0); + serde_json::to_value(reservation).expect("durable target intent snapshot") + }; + let blocked = store + .retire_scanner_pause_backlog_for_test(0, 0) + .await + .expect_err("native authority does not discharge a target mutation"); + assert!(blocked.to_string().contains("unresolved target capacity intent"), "{blocked}"); + assert_eq!(native_replica_bytes(&store.pools[0].disk_set[0]).await.0, source_bytes); + store + .retire_scanner_pause_backlog_for_test(0, 1) + .await + .expect("a different source-set revision has no responsibility for the pending mutation"); + assert_native_source_missing(&store, 1).await; + let after = { + let meta = store.pool_meta.read().await; + serde_json::to_value( + meta.pools[0] + .decommission + .as_ref() + .unwrap() + .capacity_reservation + .as_ref() + .unwrap(), + ) + .expect("retained target intent snapshot") + }; + assert_eq!(after, before, "native cleanup never clears or estimates a target mutation intent"); + shutdown_native_retirement_store(store).await; + }); + } + fn observation(now: u64, paused: bool, pending: u64) -> ScannerPauseBacklogObservation { ScannerPauseBacklogObservation { now_unix_secs: now, @@ -1618,6 +2904,38 @@ mod tests { ) } + #[test] + fn current_replica_stability_requires_every_member_to_be_seeded() { + let source = replica_id(0, 0); + let target = replica_id(1, 0); + let ledger = durable_ledger(100); + let old_record = replica_record_for_members(&ledger, &ledger, &[source]); + let old_only = crash_reload_replicas(vec![decoded_replica(source, &old_record)]); + assert!(old_only.all_current_replicas_stable()); + + let missing = ScannerPauseBacklogReplica { + id: target, + revision: Some(DataUsageCacheRevision::Missing), + state: ScannerPauseBacklogReplicaState::Missing, + }; + let stale_record = ScannerPauseBacklogReplicaRecord::new(Some(durable_ledger(10)), None); + for extra in [missing, decoded_replica(target, &stale_record)] { + let loaded = crash_reload_replicas(vec![decoded_replica(source, &old_record), extra]); + assert!(loaded.durable && loaded.stable_matches_ledger); + assert_eq!(loaded.persistence_state, "membership_repair_pending"); + assert!(!loaded.all_current_replicas_stable()); + } + + let seeded = crash_reload_replicas(vec![decoded_replica(source, &old_record), decoded_replica(target, &old_record)]); + assert_eq!(seeded.persistence_state, "membership_repair_pending"); + assert!( + seeded.all_current_replicas_stable(), + "the old proof can seed new members without claiming them" + ); + let pending = committed_scanner_pause_backlog_pending_reload(ledger, 2); + assert!(!pending.all_current_replicas_stable(), "an empty cached view cannot prove readiness"); + } + fn replica_record( stable: &ScannerPauseBacklogLedger, committed: &ScannerPauseBacklogLedger, @@ -1631,6 +2949,106 @@ mod tests { replica_record_for_members(stable, committed, &replicas) } + fn retirement_replica( + id: ScannerPauseBacklogReplicaId, + record: &ScannerPauseBacklogReplicaRecord, + ) -> ScannerPauseBacklogRetirementReplica { + ScannerPauseBacklogRetirementReplica { + pool_index: id.pool_index, + set_index: id.set_index, + data: Some(serde_json::to_vec(record).expect("native record bytes")), + } + } + + #[test] + fn native_retirement_requires_stabilization_and_valid_source_records() { + let source = replica_id(0, 0); + let targets = [replica_id(1, 0), replica_id(1, 1)]; + let old = durable_ledger(50); + let mut new = old.clone(); + new.claim_writer(100).unwrap(); + prepare_scanner_pause_backlog_persist(&mut new, 100).unwrap(); + let source_record = replica_record_for_members(&old, &old, &[source]); + let committed = replica_record_for_members(&old, &new, &targets); + let mut replicas = vec![ + retirement_replica(source, &source_record), + retirement_replica(targets[0], &committed), + retirement_replica(targets[1], &committed), + ]; + let blocked = verify_scanner_pause_backlog_retirement(0, &replicas) + .expect_err("a complete commit still needs its native stabilization barrier"); + assert!(blocked.contains("stable authority"), "{blocked}"); + + let stable = replica_record_for_members(&new, &new, &targets); + replicas[1] = retirement_replica(targets[0], &stable); + replicas[2] = retirement_replica(targets[1], &stable); + verify_scanner_pause_backlog_retirement(0, &replicas).expect("stabilized surviving native proof"); + for invalid in [b"invalid source JSON".to_vec(), br#"{"replica_schema_version":999}"#.to_vec()] { + replicas[0].data = Some(invalid); + verify_scanner_pause_backlog_retirement(0, &replicas) + .expect_err("a source with unreadable authority cannot be discarded"); + assert!(plan_scanner_pause_backlog_retirement(0, &replicas).is_err()); + } + } + + #[test] + fn native_retirement_rejects_competing_or_larger_source_commit_proofs() { + let targets = [replica_id(1, 0), replica_id(1, 1)]; + let old = durable_ledger(50); + let mut new = old.clone(); + new.claim_writer(100).unwrap(); + prepare_scanner_pause_backlog_persist(&mut new, 100).unwrap(); + let stable = replica_record_for_members(&new, &new, &targets); + for source_sets in [2, 3] { + let sources = (0..source_sets).map(|set| replica_id(0, set)).collect::>(); + let source_record = replica_record_for_members(&old, &old, &sources); + let mut replicas = sources + .iter() + .map(|id| retirement_replica(*id, &source_record)) + .collect::>(); + replicas.extend(targets.iter().map(|id| retirement_replica(*id, &stable))); + verify_scanner_pause_backlog_retirement(0, &replicas) + .expect_err("all source sets must be consulted before discarding a competing or larger native proof"); + assert!(plan_scanner_pause_backlog_retirement(0, &replicas).is_err()); + } + } + + #[test] + fn native_retirement_bootstrap_rejects_unread_members_and_independent_stable_authority() { + let source = replica_id(0, 0); + let targets = [replica_id(1, 0), replica_id(1, 1)]; + let old = durable_ledger(50); + let unread = replica_id(9, 0); + let candidate = ScannerPauseBacklogReplicaRecord::new( + None, + Some(ScannerPauseBacklogCommitRecord::new( + old.clone(), + vec![source, targets[0], targets[1], unread], + )), + ); + let mut replicas = vec![retirement_replica(source, &candidate)]; + replicas.extend(targets.iter().map(|id| ScannerPauseBacklogRetirementReplica { + pool_index: id.pool_index, + set_index: id.set_index, + data: None, + })); + let error = plan_scanner_pause_backlog_retirement(0, &replicas) + .err() + .expect("bootstrap must not turn an unread old cohort into a missing member"); + assert!(error.contains("unread"), "{error}"); + + let new = durable_ledger(100); + let source_record = replica_record_for_members(&old, &old, &[source]); + let stable = ScannerPauseBacklogReplicaRecord::new(Some(new), None); + replicas[0] = retirement_replica(source, &source_record); + replicas[1] = retirement_replica(targets[0], &stable); + replicas[2] = retirement_replica(targets[1], &stable); + let error = plan_scanner_pause_backlog_retirement(0, &replicas) + .err() + .expect("an independent source-only proof must not overwrite stable surviving authority"); + assert!(error.contains("stable authority"), "{error}"); + } + #[derive(Clone, Copy)] enum RejoinedSourceState { Missing, diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index f20374ae0..5f6ea4437 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -128,6 +128,79 @@ pub(super) mod cleanup_io_fault { } } +#[cfg(test)] +pub(super) mod recovery_intent_accept_fault { + use super::*; + + enum Fault { + Corrupt, + Running, + } + + static NEXT_ACCEPT_READBACK_FAULT: StdMutex> = StdMutex::new(None); + + pub(in crate::scanner) struct Guard; + + impl Drop for Guard { + fn drop(&mut self) { + *NEXT_ACCEPT_READBACK_FAULT + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; + } + } + + pub(in crate::scanner) fn corrupt_next_accept_readback() -> Guard { + install(Fault::Corrupt) + } + + pub(in crate::scanner) fn advance_next_accept_readback_to_running() -> Guard { + install(Fault::Running) + } + + fn install(fault: Fault) -> Guard { + let mut slot = NEXT_ACCEPT_READBACK_FAULT + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + assert!(slot.is_none(), "only one recovery intent accept readback fault may be installed"); + *slot = Some(fault); + Guard + } + + pub(super) async fn maybe_apply(storeapi: Arc, path: &str) -> Result<(), ScannerError> + where + S: ScannerObjectIO, + { + let Some(fault) = NEXT_ACCEPT_READBACK_FAULT + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take() + else { + return Ok(()); + }; + match fault { + Fault::Corrupt => save_config(storeapi, path, b"{corrupt".to_vec()).await.map_err(|err| { + ScannerError::Other(format!("failed to inject scanner recovery intent accept readback fault: {err}")) + }), + Fault::Running => { + let mut record = read_recovery_intent_record(storeapi.clone(), path).await?.ok_or_else(|| { + ScannerError::Other("scanner recovery intent disappeared before fault injection".to_string()) + })?; + record.state = SCANNER_RECOVERY_INTENT_STATE_RUNNING.to_string(); + save_config( + storeapi, + path, + serde_json::to_vec(&record) + .map_err(|err| ScannerError::Other(format!("failed to encode scanner recovery intent fault: {err}")))?, + ) + .await + .map_err(|err| { + ScannerError::Other(format!("failed to inject scanner recovery intent accept readback fault: {err}")) + }) + } + } + } +} + #[derive(Clone, Debug, Default, Serialize)] pub struct ScannerCycleRecoveryStatus { /// The immutable primary object whose revision is being guarded. @@ -564,6 +637,17 @@ fn compare_recovery_intent( } } +fn confirm_recovery_intent_acceptance( + expected: ScannerRecoveryIntentRecord, + persisted: ScannerRecoveryIntentRecord, +) -> ScannerRecoveryIntentAcceptResult { + if persisted == expected { + ScannerRecoveryIntentAcceptResult::Accepted { record: persisted } + } else { + compare_recovery_intent(&expected, persisted) + } +} + async fn read_recovery_intent_record( storeapi: Arc, path: &str, @@ -757,7 +841,20 @@ pub async fn accept_scanner_usage_recovery_intent( .map_err(|err| ScannerError::Other(format!("failed to encode scanner recovery intent: {err}")))?; match save_config_with_preconditions(storeapi.clone(), &path, encoded, DataUsageCacheRevision::Missing.preconditions()).await { - Ok(_) => Ok(ScannerRecoveryIntentAcceptResult::Accepted { record: candidate }), + Ok(_) => { + #[cfg(test)] + recovery_intent_accept_fault::maybe_apply(storeapi.clone(), &path).await?; + let persisted = match read_recovery_intent_record(storeapi.clone(), &path).await { + Ok(Some(record)) => record, + Ok(None) => { + return Err(ScannerError::Other( + "scanner recovery intent disappeared before acceptance confirmation".to_string(), + )); + } + Err(error) => return Err(error), + }; + Ok(confirm_recovery_intent_acceptance(candidate, persisted)) + } Err(EcstoreError::PreconditionFailed) => { let existing = read_recovery_intent_record(storeapi, &path).await?; let Some(existing) = existing else { diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 2b09e22d2..d9da0b619 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -61,28 +61,72 @@ async fn setup_scanner_cycle_store_with_pool_count( } async fn setup_scanner_cycle_store_at_path(root: &Path, seed_usage_baseline: bool, pool_count: usize) -> Arc { + setup_scanner_cycle_store_at_path_with_sets(root, seed_usage_baseline, pool_count, 1).await +} + +pub(super) async fn setup_scanner_cycle_store_at_path_with_sets( + root: &Path, + seed_usage_baseline: bool, + pool_count: usize, + sets_per_pool: usize, +) -> Arc { + setup_scanner_cycle_store_at_path_with_layout(root, seed_usage_baseline, pool_count, sets_per_pool, 4).await +} + +pub(super) async fn setup_scanner_cycle_store_at_path_with_layout( + root: &Path, + seed_usage_baseline: bool, + pool_count: usize, + sets_per_pool: usize, + drives_per_set: usize, +) -> Arc { + setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root, + seed_usage_baseline, + pool_count, + sets_per_pool, + drives_per_set, + true, + ) + .await +} + +pub(super) async fn setup_scanner_cycle_store_at_path_with_layout_and_disk_preinit( + root: &Path, + seed_usage_baseline: bool, + pool_count: usize, + sets_per_pool: usize, + drives_per_set: usize, + preinitialize_disks: bool, +) -> Arc { init_ecstore_config_for_scanner_tests(); let mut pools = Vec::with_capacity(pool_count); for pool_index in 0..pool_count { let mut endpoints = Vec::new(); - for disk_index in 0..4 { - let disk_path = root.join(format!("pool{pool_index}/disk{disk_index}")); - tokio::fs::create_dir_all(&disk_path) - .await - .expect("scanner cycle test disk should be created"); - let mut endpoint = - Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); - endpoint.set_pool_index(pool_index); - endpoint.set_set_index(0); - endpoint.set_disk_index(disk_index); - endpoints.push(endpoint); + for set_index in 0..sets_per_pool { + for disk_index in 0..drives_per_set { + let disk_path = if sets_per_pool == 1 { + root.join(format!("pool{pool_index}/disk{disk_index}")) + } else { + root.join(format!("pool{pool_index}/set{set_index}/disk{disk_index}")) + }; + tokio::fs::create_dir_all(&disk_path) + .await + .expect("scanner cycle test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(set_index); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } } pools.push(PoolEndpoints { legacy: false, - set_count: 1, - drives_per_set: 4, + set_count: sets_per_pool, + drives_per_set, endpoints: Endpoints::from(endpoints), - cmd_line: if pool_count == 1 { + cmd_line: if pool_count == 1 && sets_per_pool == 1 { "scanner-cycle-metrics".to_string() } else { format!("scanner-cycle-metrics-pool-{pool_index}") @@ -93,9 +137,11 @@ async fn setup_scanner_cycle_store_at_path(root: &Path, seed_usage_baseline: boo let endpoint_pools = EndpointServerPools::from(pools); let instance_ctx = Arc::new(InstanceContext::new()); instance_ctx.set_endpoints(endpoint_pools.clone()); - init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) - .await - .expect("scanner cycle test disks should initialize"); + if preinitialize_disks { + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("scanner cycle test disks should initialize"); + } let store = ECStore::new_with_instance_ctx( "127.0.0.1:0".parse().expect("test address should parse"), endpoint_pools, @@ -7634,6 +7680,25 @@ async fn scanner_cycle_confirms_lost_remote_ack_from_activity_snapshot() { "a new peer instance cannot confirm whether the old ACK reached durable dirty state" ); + let mut stale_activity = scanner_node_activity("epoch-a", 7, 3); + stale_activity.dirty_usage_generation = 4; + let stale_clean_activity = BTreeMap::from([("node-2".to_string(), stale_activity)]); + let stale_clean = remote_dirty_usage_acknowledgement_pending( + 8, + 1, + std::slice::from_ref(&acknowledgement), + std::future::ready(Err::(std::io::Error::other( + "response lost before newer generation was observed", + ))), + || async { Ok(stale_clean_activity) }, + ) + .await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, stale_clean), + ScannerCycleOutcome::CompletedWithPendingMaintenance, + "a clean peer snapshot from before the acknowledged generation cannot prove the ACK reached durable dirty state" + ); + let mut written_activity = scanner_node_activity("epoch-a", 7, 3); written_activity.dirty_usage_generation = 6; written_activity.dirty_usage_pending = true; @@ -7707,6 +7772,47 @@ async fn scanner_cycle_confirms_lost_scoped_ack_only_after_same_instance_clean_a "a restarted peer cannot prove the scoped ACK reached the old scanner instance" ); + let mut stale_activity = scanner_node_activity("epoch-a", 7, 3); + stale_activity.dirty_usage_generation = 4; + let stale_clean_activity = BTreeMap::from([("node-2".to_string(), stale_activity)]); + let stale_clean = remote_dirty_usage_acknowledgement_pending( + 8, + 1, + std::slice::from_ref(&acknowledgement), + std::future::ready(Err::(std::io::Error::other( + "scoped ACK transport failed before the requested generation was observed", + ))), + || async { Ok(stale_clean_activity) }, + ) + .await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, stale_clean), + ScannerCycleOutcome::CompletedWithPendingMaintenance, + "a clean peer snapshot from before the scoped ACK generation cannot prove the ACK reached durable dirty state" + ); + + let empty_scoped_ack = ScannerDirtyUsageAcknowledgement { + host: "node-2".to_string(), + instance_id: "epoch-a".to_string(), + kind: ScannerDirtyUsageAcknowledgementKind::Scoped { + owner_id: Uuid::from_u128(0x11111111111111111111111111111111).to_string(), + entries: Vec::new(), + }, + }; + let empty_scoped_clean = remote_dirty_usage_acknowledgement_pending( + 8, + 1, + &[empty_scoped_ack], + std::future::ready(Err::(std::io::Error::other("empty scoped ACK failed before peer delivery"))), + || async { Ok(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])) }, + ) + .await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, empty_scoped_clean), + ScannerCycleOutcome::CompletedWithPendingMaintenance, + "an empty scoped ACK has no durable generation to reconcile after response loss" + ); + let mut written_activity = scanner_node_activity("epoch-a", 7, 3); written_activity.dirty_usage_generation = 6; written_activity.dirty_usage_pending = true; @@ -9211,6 +9317,64 @@ fn post_lease_activity_proof_rejects_a_put_tail_that_finished_before_lease_acqui ); } +#[test] +fn remote_lease_validation_failure_without_movement_debt_is_activity_baseline_unavailable() { + let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let lease_targets = scanner_activity_publication_lease_targets(&before); + let mut after = before.clone(); + after + .get_mut("node-2") + .expect("writer should be present") + .namespace_generation += 1; + + assert_eq!( + before["node-2"].movement_generation, after["node-2"].movement_generation, + "ordinary namespace writes must not be reported as movement" + ); + assert!(scanner_activity_allows_usage_publication(&after)); + assert_eq!( + scanner_remote_publication_lease_failure_defer_reason(&lease_targets, true, Ok(after)), + ScannerCycleDeferReason::ActivityBaselineUnavailable + ); +} + +#[test] +fn remote_lease_validation_failure_preserves_movement_defer_for_remote_fence_loss() { + let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let lease_targets = scanner_activity_publication_lease_targets(&before); + + let mut movement_changed = before.clone(); + movement_changed + .get_mut("node-2") + .expect("writer should be present") + .movement_generation += 1; + assert_eq!( + scanner_remote_publication_lease_failure_defer_reason(&lease_targets, true, Ok(movement_changed)), + ScannerCycleDeferReason::DataMovement + ); + + let mut restarted = before.clone(); + restarted.get_mut("node-2").expect("writer should be present").instance_id = "epoch-b".to_string(); + assert_eq!( + scanner_remote_publication_lease_failure_defer_reason(&lease_targets, true, Ok(restarted)), + ScannerCycleDeferReason::DataMovement + ); + + let mut blocked = before.clone(); + blocked + .get_mut("node-2") + .expect("writer should be present") + .publication_blocked = true; + assert_eq!( + scanner_remote_publication_lease_failure_defer_reason(&lease_targets, true, Ok(blocked)), + ScannerCycleDeferReason::DataMovement + ); + assert_eq!( + scanner_remote_publication_lease_failure_defer_reason(&lease_targets, false, Ok(before)), + ScannerCycleDeferReason::DataMovement + ); +} + #[test] fn post_lease_activity_proof_requires_a_complete_matching_baseline() { let before = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); diff --git a/crates/scanner/src/scanner/tests/recovery_control.rs b/crates/scanner/src/scanner/tests/recovery_control.rs index 9191c238c..61ac96bc5 100644 --- a/crates/scanner/src/scanner/tests/recovery_control.rs +++ b/crates/scanner/src/scanner/tests/recovery_control.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::cycle_state::cleanup_io_fault; +use super::super::cycle_state::{cleanup_io_fault, recovery_intent_accept_fault}; use super::*; use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats}; @@ -256,6 +256,100 @@ async fn scanner_recovery_intent_accept_is_durable_and_idempotent() { assert_eq!(replay, ScannerRecoveryIntentAcceptResult::Replayed { record }); } +#[tokio::test] +#[serial] +async fn scanner_recovery_intent_accept_requires_confirmed_readback() { + let (_dir, store) = setup_scanner_cycle_store().await; + let _fault = recovery_intent_accept_fault::corrupt_next_accept_readback(); + let error = + accept_scanner_usage_recovery_intent(store.clone(), recovery_intent_request("intent-key-0001-readback", "operator-a")) + .await + .expect_err("accept must fail when the just-written intent cannot be confirmed"); + assert!(error.to_string().contains("scanner recovery intent is invalid"), "{error}"); + + let restarted = restart_scanner_cycle_store_from(&store).await; + let error = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted) + .await + .expect_err("unconfirmed corrupt intent must remain a fail-closed startup error"); + assert!(error.to_string().contains("scanner recovery intent is invalid"), "{error}"); +} + +#[tokio::test] +#[serial] +async fn scanner_recovery_intent_accept_replays_if_execution_advances_before_readback() { + let (_dir, store) = setup_scanner_cycle_store().await; + let request = recovery_intent_request("intent-key-0001-running", "operator-a"); + let _fault = recovery_intent_accept_fault::advance_next_accept_readback_to_running(); + let replay = accept_scanner_usage_recovery_intent(store.clone(), request.clone()) + .await + .expect("same request advanced by execution remains idempotent"); + let record = match replay { + ScannerRecoveryIntentAcceptResult::Replayed { record } => record, + other => panic!("advanced same-request record must replay instead of conflict: {other:?}"), + }; + assert_eq!(record.state, "running"); + + let retry = accept_scanner_usage_recovery_intent(store, request) + .await + .expect("lost response retry observes the running durable record"); + assert_eq!(retry, ScannerRecoveryIntentAcceptResult::Replayed { record }); +} + +#[tokio::test] +#[serial] +async fn concurrent_scanner_recovery_intent_acceptance_uses_one_durable_record() { + let (_dir, store) = setup_scanner_cycle_store().await; + let request = recovery_intent_request("intent-key-0001-concurrent", "operator-a"); + let mut tasks = Vec::new(); + for _ in 0..12 { + let store = store.clone(); + let request = request.clone(); + tasks.push(tokio::spawn(async move { + accept_scanner_usage_recovery_intent(store, request) + .await + .expect("concurrent same-key accept should converge") + })); + } + + let mut accepted = 0usize; + let mut replayed = 0usize; + let mut records = Vec::new(); + for task in tasks { + match task.await.expect("accept task should not panic") { + ScannerRecoveryIntentAcceptResult::Accepted { record } => { + accepted += 1; + records.push(record); + } + ScannerRecoveryIntentAcceptResult::Replayed { record } => { + replayed += 1; + records.push(record); + } + other => panic!("same-key accepts must not conflict: {other:?}"), + } + } + assert_eq!(accepted, 1, "exactly one request may win the missing-record CAS"); + assert_eq!(replayed, 11, "all other same-key requests must replay the durable winner"); + assert!( + records.windows(2).all(|pair| pair[0] == pair[1]), + "all accepts must return the same durable identity" + ); + + let restarted = restart_scanner_cycle_store_from(&store).await; + let replayable = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted.clone()) + .await + .expect("startup should rediscover the single indexed intent"); + assert_eq!(replayable, vec![records[0].intent_id.clone()]); + let replay = accept_scanner_usage_recovery_intent(restarted, request) + .await + .expect("lost response after restart should replay the same record"); + assert_eq!( + replay, + ScannerRecoveryIntentAcceptResult::Replayed { + record: records[0].clone() + } + ); +} + #[tokio::test] #[serial] async fn scanner_recovery_intent_executor_persists_completed_progress() { @@ -457,7 +551,7 @@ async fn scanner_recovery_intent_startup_rejects_corrupt_pending_record() { #[tokio::test] #[serial] -async fn scanner_recovery_intent_disabled_startup_preserves_non_terminal_intent() { +async fn scanner_recovery_intent_disabled_startup_replays_non_terminal_intent() { let (_dir, store) = setup_scanner_cycle_store().await; let record = match accept_scanner_usage_recovery_intent( store.clone(), @@ -473,12 +567,12 @@ async fn scanner_recovery_intent_disabled_startup_preserves_non_terminal_intent( let restarted = restart_scanner_cycle_store_from(&store).await; run_disabled_startup(CancellationToken::new(), restarted.clone()).await; - let preserved = get_scanner_usage_recovery_intent(restarted, &record.intent_id) + let completed = get_scanner_usage_recovery_intent(restarted, &record.intent_id) .await - .expect("startup-skipped intent should read") - .expect("startup-skipped intent should remain durable"); - assert_eq!(preserved.state, "accepted"); - assert_eq!(preserved.intent_id, record.intent_id); + .expect("startup-replayed intent should read") + .expect("startup-replayed intent should remain durable"); + assert_eq!(completed.state, "completed"); + assert_eq!(completed.intent_id, record.intent_id); } #[tokio::test] @@ -547,6 +641,17 @@ async fn scanner_recovery_intent_query_rejects_corrupt_or_unknown_records() { .await .expect_err("corrupt intent must not decode as absent"); assert!(error.to_string().contains("scanner recovery intent is invalid")); + + let mut future = serde_json::to_value(&record).expect("record value"); + future["future_writer_capability"] = serde_json::json!("durable-accept-v2"); + save_config(store.clone(), &path, serde_json::to_vec(&future).expect("future record should encode")) + .await + .expect("future durable record"); + let error = get_scanner_usage_recovery_intent(store.clone(), &record.intent_id) + .await + .expect_err("future writer payload must not decode as a known terminal state"); + assert!(error.to_string().contains("scanner recovery intent is invalid")); + let unknown = get_scanner_usage_recovery_intent(store, &scanner_recovery_actor_sha256("missing")) .await .expect("missing intent should read as absent"); diff --git a/crates/scanner/src/scanner/tests/scoped_ack_publication.rs b/crates/scanner/src/scanner/tests/scoped_ack_publication.rs index 2fa58fff2..472e595d7 100644 --- a/crates/scanner/src/scanner/tests/scoped_ack_publication.rs +++ b/crates/scanner/src/scanner/tests/scoped_ack_publication.rs @@ -432,7 +432,12 @@ async fn scoped_ack_publication_stale_baseline_cannot_prove_a_replaced_root() { #[tokio::test] #[serial] async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish() { - for mutation in ["remote_ack_target", "publication_epoch", "remote_lease_targets"] { + for mutation in [ + "remote_ack_target", + "remote_scoped_ack_target", + "publication_epoch", + "remote_lease_targets", + ] { let (_directory, store) = candidate_store().await; let (scan, candidate) = complete_candidate(&store, PROOF_CYCLE).await; let baseline = read_data_usage_persist_baseline(store.clone()) @@ -465,6 +470,18 @@ async fn scoped_ack_publication_rejects_builder_mutation_after_real_root_publish instance_id: crate::scanner_activity_epoch().to_string(), kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Generation(changed_generation), }]), + "remote_scoped_ack_target" => scan.with_remote_dirty_usage_acknowledgements(vec![ScannerDirtyUsageAcknowledgement { + host: "proof-peer:9000".to_string(), + instance_id: crate::scanner_activity_epoch().to_string(), + kind: crate::scanner::ScannerDirtyUsageAcknowledgementKind::Scoped { + owner_id: crate::scanner_activity_epoch().to_string(), + entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry { + bucket: PROOF_BUCKET.to_string(), + bucket_incarnation: uuid::Uuid::from_u128(0x11111111111111111111111111111111), + generation: changed_generation, + }], + }, + }]), "publication_epoch" => scan.with_publication_epoch(Some(changed_epoch)), "remote_lease_targets" => scan.with_remote_publication_lease_targets(vec![( "proof-peer:9000".to_string(), diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs index 917850693..b3b7cd72d 100644 --- a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs @@ -3,7 +3,8 @@ use super::*; use crate::segment_invalidation::{ MAX_SEGMENT_INVALIDATION_BYTES, MAX_SEGMENT_INVALIDATION_ENTRIES, SegmentInvalidationDomain, SegmentInvalidationEnvelope, - SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProof, admit_segment_invalidation, + SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProducerIdentity, SegmentInvalidationProof, + admit_segment_invalidation, complete_segment_invalidation_producers, }; use std::collections::BTreeSet; @@ -11,7 +12,8 @@ const MAX_WALK_SAMPLES: usize = 32; const MAX_WALK_BYTES: usize = 1024; fn segment_producers() -> BTreeSet { - SegmentInvalidationProducer::REQUIRED.into_iter().collect() + complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION) + .expect("fixture should enumerate the complete production producer matrix") } fn segment_envelope() -> SegmentInvalidationEnvelope { diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index a9f4e8eab..69603627a 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -39,7 +39,7 @@ use s3s::dto::{ BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration, }; use sha2::{Digest as _, Sha256}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::future::Future; use std::path::Path; use std::pin::Pin; @@ -98,6 +98,28 @@ const METRIC_SCANNER_SET_SCANS_QUEUED: &str = "rustfs_scanner_set_scans_queued"; const METRIC_SCANNER_DISK_BUCKET_SCANS_ACTIVE: &str = "rustfs_scanner_disk_bucket_scans_active"; const METRIC_SCANNER_DISK_BUCKET_SCANS_QUEUED: &str = "rustfs_scanner_disk_bucket_scans_queued"; +pub(crate) const SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS: [&str; 9] = [ + "source", + "bucket_incarnation", + "key_format", + "baseline_scan_plan_digest", + "process_epoch", + "generation_window", + "producer_identities", + "durable_dirty_producer_journal", + "ack_generation_guard", +]; +pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = [ + "missing_producer_identity", + "missing_durable_journal_replay", + "restart_gap", + "generation_gap", + "overflow", + "stale_ack_generation", + "missing_cold_zero_walk_oracle", + "distributed_without_peer_invalidation", +]; + pub type DirtyUsageBuckets = HashMap; #[derive(Clone, Debug)] @@ -163,11 +185,54 @@ struct ScannerPeerDirtyUsageExpectation { struct VerifiedRemoteDirtyUsage { dirty_buckets: HashSet, acknowledgements: Vec, + peer_count: usize, + dirty_peer_count: usize, } struct ScannerBucketScopeResolutionResult { scope: ScannerBucketScanScope, remote_dirty_usage_acknowledgements: Vec, + distributed_segment_invalidation_evidence: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct DistributedSegmentInvalidationEvidence { + pub(crate) invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain, + pub(crate) distributed_ec_invalidation: bool, + pub(crate) peer_count: usize, + pub(crate) dirty_peer_count: usize, + pub(crate) same_window_remote_proof: bool, + pub(crate) all_peers_bound_to_generation_window: bool, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct ScannerSegmentReuseActivationProof { + pub(crate) production_activation: bool, + pub(crate) producer_identity_coverage_complete: bool, + pub(crate) durable_producer_identity: bool, + pub(crate) durable_dirty_producer_journal: bool, + pub(crate) restart_gap_absent: bool, + pub(crate) generation_window_bound: bool, + pub(crate) overflow_absent: bool, + pub(crate) ack_generation_guard: bool, + pub(crate) cold_zero_walk_oracle: bool, + pub(crate) distributed_peer_invalidation: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ScannerSegmentReuseActivationPreflight { + pub(crate) production_activation: bool, + pub(crate) scanner_segment_reuse_activated: bool, + pub(crate) proof_inputs: &'static [&'static str], + pub(crate) fail_closed_checks: &'static [&'static str], + pub(crate) fail_closed_blockers: [Option<&'static str>; 8], +} + +impl ScannerSegmentReuseActivationPreflight { + #[cfg(test)] + pub(crate) fn fail_closed_blockers(&self) -> impl Iterator + '_ { + self.fail_closed_blockers.iter().filter_map(|blocker| *blocker) + } } fn verified_remote_dirty_usage( @@ -191,6 +256,7 @@ fn verified_remote_dirty_usage( || !snapshot.complete || snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX) || (expected.pending && snapshot.pending_bucket_count == 0) + || (!expected.pending && snapshot.pending_bucket_count != 0) { return None; } @@ -216,9 +282,13 @@ fn verified_remote_dirty_usage( } } + let peer_count = received_peers.len(); + let dirty_peer_count = acknowledgements.len(); (received_peers.len() == expected_peers.len()).then_some(VerifiedRemoteDirtyUsage { dirty_buckets, acknowledgements, + peer_count, + dirty_peer_count, }) } @@ -236,7 +306,7 @@ fn scanner_scoped_dirty_usage_ack_exceeds_cost_threshold( fn resolve_remote_dirty_usage_scope( requested_scope: ScannerBucketScanScope, - mut dirty_buckets: HashSet, + dirty_usage_snapshot: &DirtyUsageSnapshot, remote_dirty_usage: VerifiedRemoteDirtyUsage, all_buckets: &[BucketInfo], baseline_proof: ScannerCacheBaselineProof<'_>, @@ -244,14 +314,26 @@ fn resolve_remote_dirty_usage_scope( let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult { scope, remote_dirty_usage_acknowledgements: Vec::new(), + distributed_segment_invalidation_evidence: None, }; + let peer_count = remote_dirty_usage.peer_count; + let dirty_peer_count = remote_dirty_usage.dirty_peer_count; + let remote_dirty_buckets = remote_dirty_usage.dirty_buckets.clone(); + let mut dirty_buckets = dirty_usage_snapshot.buckets.keys().cloned().collect::>(); dirty_buckets.extend(remote_dirty_usage.dirty_buckets); // Peer snapshots contribute bucket names only; the local prefix scopes // would narrow a bucket a peer dirtied elsewhere, so the merged scope // stays at bucket granularity (same rule as the local fallthrough). - let scope = - scoped_scan_scope_from_dirty_buckets(requested_scope, dirty_buckets, None, true, false, all_buckets, baseline_proof); + let scope = scoped_scan_scope_from_dirty_buckets( + requested_scope.clone(), + dirty_buckets.clone(), + None, + true, + false, + all_buckets, + baseline_proof, + ); if scope.is_default() { return default_result(scope); } @@ -283,10 +365,47 @@ fn resolve_remote_dirty_usage_scope( if scanner_scoped_dirty_usage_ack_exceeds_cost_threshold(&scoped_acknowledgements) { return default_result(ScannerBucketScanScope::default()); } + let has_scoped_acknowledgements = !scoped_acknowledgements.is_empty(); + + let distributed_segment_invalidation_evidence = + (dirty_peer_count > 0 && has_scoped_acknowledgements).then_some(DistributedSegmentInvalidationEvidence { + invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc, + distributed_ec_invalidation: true, + peer_count, + dirty_peer_count, + same_window_remote_proof: true, + all_peers_bound_to_generation_window: true, + }); + let segment_reuse_activation_preflight = scanner_segment_reuse_activation_preflight_for_baseline_with_evidence( + dirty_usage_snapshot, + true, + baseline_proof, + distributed_segment_invalidation_evidence, + ); + let scope = if segment_reuse_activation_preflight.scanner_segment_reuse_activated { + let local_only_scopes = dirty_usage_snapshot + .scopes + .iter() + .filter(|(bucket, _)| !remote_dirty_buckets.contains(bucket.as_str())) + .map(|(bucket, scope)| (bucket.clone(), scope.clone())) + .collect::(); + scoped_scan_scope_from_dirty_buckets( + requested_scope, + dirty_buckets, + (!local_only_scopes.is_empty()).then_some(&local_only_scopes), + true, + true, + all_buckets, + baseline_proof, + ) + } else { + scope + }; ScannerBucketScopeResolutionResult { scope, remote_dirty_usage_acknowledgements: scoped_acknowledgements, + distributed_segment_invalidation_evidence, } } @@ -358,6 +477,63 @@ fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof< complete_scanner_cache_snapshot_plan_digest(&observed, proof, false) } +fn scanner_segment_reuse_baseline_producer_evidence( + dirty_usage_snapshot: &DirtyUsageSnapshot, + baseline_proof: ScannerCacheBaselineProof<'_>, +) -> (DirtyUsageProducerEvidence, bool) { + let mut evidence = dirty_usage_producer_evidence(dirty_usage_snapshot); + if !evidence.generation_window_bound || !evidence.producer_identity_coverage_complete { + return (evidence, false); + } + + let Some(authoritative_data) = baseline_proof.authoritative_data else { + return (evidence, false); + }; + let Ok(authoritative) = serde_json::from_slice::(authoritative_data) else { + return (evidence, false); + }; + if complete_scanner_cache_snapshot_plan_digest(&authoritative, baseline_proof, true).is_none() { + return (evidence, false); + } + let Some(cold_zero_walk_oracle) = + scanner_snapshot_set_states_segment_reuse_proof(&authoritative, &evidence, baseline_proof.expected_sources) + else { + return (evidence, false); + }; + + evidence.durable_producer_identity = true; + evidence.restart_gap_absent = true; + (evidence, cold_zero_walk_oracle) +} + +fn scanner_snapshot_set_states_segment_reuse_proof( + snapshot: &DataUsageInfo, + evidence: &DirtyUsageProducerEvidence, + expected_sources: &HashSet, +) -> Option { + let mut covered_sources = HashSet::with_capacity(expected_sources.len()); + let mut cold_zero_walk_oracle = true; + let all_sets_proved = snapshot.usage_snapshot_set_states.iter().all(|state| { + let (Ok(pool_index), Ok(set_index)) = (usize::try_from(state.pool_index), usize::try_from(state.set_index)) else { + return false; + }; + let source = DataUsageCacheSource::new(pool_index, set_index); + let proved = state.complete + && !state.tombstone + && expected_sources.contains(&source) + && covered_sources.insert(source) + && scanner_segment_invalidation_baseline_proof_matches(state.segment_invalidation_proof.as_ref(), evidence); + if proved { + cold_zero_walk_oracle &= state + .segment_invalidation_proof + .as_ref() + .is_some_and(|proof| proof.cold_zero_walk_oracle); + } + proved + }); + (all_sets_proved && covered_sources.len() == expected_sources.len()).then_some(cold_zero_walk_oracle) +} + fn scoped_scan_scope_from_dirty_buckets( requested_scope: ScannerBucketScanScope, dirty_buckets: HashSet, @@ -406,10 +582,184 @@ fn scoped_scan_scope_from_dirty_buckets( ScannerBucketScanScope::from_dirty_buckets(selected_buckets, selected_bucket_prefixes, baseline_scan_plan_digest) } +fn scanner_segment_reuse_activation_preflight() -> ScannerSegmentReuseActivationPreflight { + scanner_segment_reuse_activation_preflight_from_proof(ScannerSegmentReuseActivationProof { + production_activation: true, + ..Default::default() + }) +} + +fn scanner_segment_reuse_activation_preflight_from_proof( + proof: ScannerSegmentReuseActivationProof, +) -> ScannerSegmentReuseActivationPreflight { + ScannerSegmentReuseActivationPreflight { + production_activation: proof.production_activation, + scanner_segment_reuse_activated: proof.production_activation + && proof.producer_identity_coverage_complete + && proof.durable_producer_identity + && proof.durable_dirty_producer_journal + && proof.restart_gap_absent + && proof.generation_window_bound + && proof.overflow_absent + && proof.ack_generation_guard + && proof.cold_zero_walk_oracle + && proof.distributed_peer_invalidation, + proof_inputs: &SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS, + fail_closed_checks: &SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS, + fail_closed_blockers: [ + (!proof.producer_identity_coverage_complete || !proof.durable_producer_identity) + .then_some("missing_producer_identity"), + (!proof.durable_dirty_producer_journal).then_some("missing_durable_journal_replay"), + (!proof.restart_gap_absent).then_some("restart_gap"), + (!proof.generation_window_bound).then_some("generation_gap"), + (!proof.overflow_absent).then_some("overflow"), + (!proof.ack_generation_guard).then_some("stale_ack_generation"), + (!proof.cold_zero_walk_oracle).then_some("missing_cold_zero_walk_oracle"), + (!proof.distributed_peer_invalidation).then_some("distributed_without_peer_invalidation"), + ], + } +} + +fn scanner_segment_reuse_activation_preflight_for_cycle( + dirty_usage_snapshot: &DirtyUsageSnapshot, + dirty_usage_producer_evidence: DirtyUsageProducerEvidence, + distributed: bool, + distributed_segment_invalidation_evidence: Option, + cold_zero_walk_oracle: bool, +) -> ScannerSegmentReuseActivationPreflight { + scanner_segment_reuse_activation_preflight_from_proof(ScannerSegmentReuseActivationProof { + production_activation: true, + producer_identity_coverage_complete: dirty_usage_producer_evidence.producer_identity_coverage_complete, + durable_producer_identity: dirty_usage_producer_evidence.durable_producer_identity, + durable_dirty_producer_journal: dirty_usage_producer_evidence.durable_dirty_producer_journal, + restart_gap_absent: dirty_usage_producer_evidence.restart_gap_absent, + generation_window_bound: dirty_usage_snapshot.covers_all_pending + && dirty_usage_snapshot.generation != 0 + && dirty_usage_snapshot.generation != u64::MAX + && dirty_usage_producer_evidence.generation_window_bound, + overflow_absent: dirty_usage_snapshot.covers_all_pending, + ack_generation_guard: true, + cold_zero_walk_oracle, + distributed_peer_invalidation: scanner_distributed_segment_invalidation_admitted( + distributed, + distributed_segment_invalidation_evidence, + ), + }) +} + +fn scanner_segment_reuse_activation_preflight_for_baseline( + dirty_usage_snapshot: &DirtyUsageSnapshot, + distributed: bool, + baseline_proof: ScannerCacheBaselineProof<'_>, +) -> ScannerSegmentReuseActivationPreflight { + scanner_segment_reuse_activation_preflight_for_baseline_with_evidence(dirty_usage_snapshot, distributed, baseline_proof, None) +} + +fn scanner_segment_reuse_activation_preflight_for_baseline_with_evidence( + dirty_usage_snapshot: &DirtyUsageSnapshot, + distributed: bool, + baseline_proof: ScannerCacheBaselineProof<'_>, + distributed_segment_invalidation_evidence: Option, +) -> ScannerSegmentReuseActivationPreflight { + let (dirty_usage_producer_evidence, cold_zero_walk_oracle) = + scanner_segment_reuse_baseline_producer_evidence(dirty_usage_snapshot, baseline_proof); + scanner_segment_reuse_activation_preflight_for_cycle( + dirty_usage_snapshot, + dirty_usage_producer_evidence, + distributed, + distributed_segment_invalidation_evidence, + cold_zero_walk_oracle, + ) +} + +fn scanner_distributed_segment_invalidation_admitted( + distributed: bool, + evidence: Option, +) -> bool { + if !distributed { + return true; + } + evidence.is_some_and(|evidence| { + evidence.invalidation_domain == crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc + && evidence.distributed_ec_invalidation + && evidence.same_window_remote_proof + && evidence.all_peers_bound_to_generation_window + && evidence.dirty_peer_count > 0 + && evidence.dirty_peer_count <= evidence.peer_count + }) +} + +fn scanner_durable_segment_invalidation_evidence( + dirty_usage_snapshot: &DirtyUsageSnapshot, + results: &[DataUsageCache], + expected_sources: &HashSet, +) -> DirtyUsageProducerEvidence { + let mut evidence = dirty_usage_producer_evidence(dirty_usage_snapshot); + if !evidence.generation_window_bound + || !evidence.producer_identity_coverage_complete + || !scanner_results_form_complete_snapshot(results, expected_sources) + { + return evidence; + } + + let mut covered_sources = HashSet::with_capacity(expected_sources.len()); + let all_sets_proved = results.iter().all(|result| { + let Some(source) = result.info.source else { + return false; + }; + expected_sources.contains(&source) + && covered_sources.insert(source) + && scanner_segment_invalidation_proof_matches(result.info.segment_invalidation_proof.as_ref(), &evidence) + }); + if all_sets_proved && covered_sources.len() == expected_sources.len() { + evidence.durable_producer_identity = true; + evidence.restart_gap_absent = true; + } + evidence +} + +fn scanner_segment_invalidation_proof_matches( + proof: Option<&crate::DataUsageSegmentInvalidationProof>, + evidence: &DirtyUsageProducerEvidence, +) -> bool { + proof.is_some_and(|proof| { + scanner_segment_invalidation_proof_is_well_formed(proof) + && proof.generation_start == evidence.generation_start + && proof.generation_end == evidence.generation_end + }) +} + +fn scanner_segment_invalidation_baseline_proof_matches( + proof: Option<&crate::DataUsageSegmentInvalidationProof>, + evidence: &DirtyUsageProducerEvidence, +) -> bool { + proof.is_some_and(|proof| { + scanner_segment_invalidation_proof_is_well_formed(proof) + && ((proof.generation_start == evidence.generation_start && proof.generation_end == evidence.generation_end) + || proof.generation_end < evidence.generation_start) + }) +} + +fn scanner_segment_invalidation_proof_is_well_formed(proof: &crate::DataUsageSegmentInvalidationProof) -> bool { + proof.process_epoch == scanner_activity_epoch() + && proof.generation_start != 0 + && proof.generation_end >= proof.generation_start + && proof.producer_identity_coverage_complete +} + +fn scanner_completed_set_segment_invalidation_proof( + proof: &Option, + cold_zero_walk_reuse_candidate: bool, +) -> Option { + proof.clone().map(|mut proof| { + proof.cold_zero_walk_oracle = cold_zero_walk_reuse_candidate; + proof + }) +} + +#[cfg(test)] fn scanner_segment_reuse_activated() -> bool { - // Production segment reuse stays disabled until a durable mutation-stream - // proof satisfies the segment invalidation contract. - false + scanner_segment_reuse_activation_preflight().scanner_segment_reuse_activated } pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool { @@ -471,6 +821,8 @@ pub struct ScannerBucketScanPlan { bucket_failures: ScannerBucketFailureState, pending_maintenance_work: Arc, cache_cycle_floor: Arc, + cold_zero_walk_reuse_observed: Arc, + segment_invalidation_proof: Option, } #[derive(Clone, Default)] @@ -603,6 +955,25 @@ fn scanner_bucket_scan_status(has_failed: bool, has_partial: bool, has_namespace } } +fn scanner_cycle_cold_zero_walk_oracle( + scan_scope: &ScannerBucketScanScope, + all_buckets: &[BucketInfo], + completed_all_sets: bool, + scan_scope_matches: bool, + bucket_scan_status: ScannerBucketScanStatus, + cold_zero_walk_reuse_observed: bool, +) -> bool { + let Some(selected_buckets) = scan_scope.selected_buckets.as_deref() else { + return false; + }; + cold_zero_walk_reuse_observed + && !selected_buckets.is_empty() + && completed_all_sets + && scan_scope_matches + && bucket_scan_status == ScannerBucketScanStatus::Complete + && all_buckets.iter().any(|bucket| !selected_buckets.contains(&bucket.name)) +} + fn classify_nsscanner_cycle( completed_all_sets: bool, budget_elapsed: bool, @@ -1056,6 +1427,8 @@ pub(crate) struct ScannerCycleResult { observational_snapshot_published: bool, dirty_usage_clear: Option, remote_dirty_usage_acknowledgements: Vec, + distributed_segment_invalidation_evidence: Option, + segment_reuse_activation_preflight: ScannerSegmentReuseActivationPreflight, remote_publication_lease_targets: Vec<(String, String, u64)>, failed_dirty_usage: bool, pending_maintenance_work: bool, @@ -1072,6 +1445,8 @@ impl ScannerCycleResult { observational_snapshot_published: false, dirty_usage_clear, remote_dirty_usage_acknowledgements: Vec::new(), + distributed_segment_invalidation_evidence: None, + segment_reuse_activation_preflight: scanner_segment_reuse_activation_preflight(), remote_publication_lease_targets: Vec::new(), failed_dirty_usage: false, pending_maintenance_work: false, @@ -1137,6 +1512,21 @@ impl ScannerCycleResult { self } + fn with_distributed_segment_invalidation_evidence( + mut self, + evidence: Option, + ) -> Self { + self.publication_expectation = None; + self.distributed_segment_invalidation_evidence = evidence; + self + } + + fn with_segment_reuse_activation_preflight(mut self, preflight: ScannerSegmentReuseActivationPreflight) -> Self { + self.publication_expectation = None; + self.segment_reuse_activation_preflight = preflight; + self + } + pub(crate) fn with_remote_publication_lease_targets(mut self, targets: Vec<(String, String, u64)>) -> Self { self.publication_expectation = None; self.remote_publication_lease_targets = targets; @@ -1222,10 +1612,14 @@ pub(crate) use cache::{ current_cache_root_or_prepare_with_generation, }; pub use dirty_usage::{ - ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, - acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket, - record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, - scanner_dirty_usage_state, scanner_maintenance_generation, + ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver, + ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayError, + ScannerDurableDirtyUsageReplayRecord, ScannerDurableDirtyUsageReplayScope, acknowledge_dirty_usage_generation, + acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, encode_durable_dirty_usage_producer_replay_record, + record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer, record_dirty_usage_bucket_from_producers, + record_dirty_usage_object, record_dirty_usage_object_from_producer, record_scanner_maintenance_change, + replay_durable_dirty_usage_producer_record, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, + scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer, }; #[cfg(test)] pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests}; diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 328b91ce9..2eb88089f 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -464,6 +464,7 @@ pub(super) fn completed_usage_candidate( scan_plan_digest: Some(result.info.scan_plan_digest?.0), complete: true, tombstone: false, + segment_invalidation_proof: result.info.segment_invalidation_proof.clone(), }) }) .collect::>>()?; @@ -642,13 +643,14 @@ pub(super) fn observational_data_usage_info( let current_snapshot = current.is_some(); let selected = current.or(lkg); if let Some(selected) = selected { - let (cycle, epoch, digest, last_update, complete) = if current_snapshot { + let (cycle, epoch, digest, last_update, complete, segment_invalidation_proof) = if current_snapshot { ( Some(selected.info.next_cycle), Some(selected.info.leader_epoch), selected.info.scan_plan_digest.map(|digest| digest.0), selected.info.last_update, true, + selected.info.segment_invalidation_proof.clone(), ) } else { ( @@ -657,6 +659,7 @@ pub(super) fn observational_data_usage_info( selected.info.lkg_scan_plan_digest.map(|digest| digest.0), selected.info.lkg_last_update, false, + None, ) }; set_states.push(DataUsageSnapshotSetState { @@ -667,6 +670,7 @@ pub(super) fn observational_data_usage_info( scan_plan_digest: digest, complete, tombstone: false, + segment_invalidation_proof, }); usable.push((selected, last_update)); } else { @@ -678,6 +682,7 @@ pub(super) fn observational_data_usage_info( scan_plan_digest: Some(expected_plan_digest.0), complete: false, tombstone: false, + segment_invalidation_proof: None, }); } } diff --git a/crates/scanner/src/scanner_io/dirty_usage.rs b/crates/scanner/src/scanner_io/dirty_usage.rs index 6457f9585..2c0e7fe4e 100644 --- a/crates/scanner/src/scanner_io/dirty_usage.rs +++ b/crates/scanner/src/scanner_io/dirty_usage.rs @@ -13,19 +13,34 @@ // limitations under the License. /// process-wide dirty-usage invalidation state, its acknowledgment protocol, and snapshot helpers. use super::*; +use std::collections::BTreeSet; +use std::sync::RwLock as StdRwLock; pub(super) static DIRTY_USAGE_BUCKET_GENERATION: AtomicU64 = AtomicU64::new(0); pub(super) static DIRTY_USAGE_BUCKETS: LazyLock> = LazyLock::new(|| StdMutex::new(HashMap::new())); -// Lock order when both dirty maps are needed is `DIRTY_USAGE_BUCKETS` followed -// by `DIRTY_USAGE_BUCKET_SCOPES`. Both are held only for synchronous map -// updates, so no scanner task can observe a bucket generation without its -// matching scope. +// Lock order when dirty usage state is updated is `DIRTY_USAGE_BUCKETS`, +// `DIRTY_USAGE_BUCKET_SCOPES`, then `DIRTY_USAGE_PRODUCER_IDENTITIES`. All +// guards are held only for synchronous map updates, so no scanner task can +// observe a bucket generation without its matching scope and producer evidence. pub(super) static DIRTY_USAGE_BUCKET_SCOPES: LazyLock> = LazyLock::new(|| StdMutex::new(HashMap::new())); +// Non-authoritative process-local producer coverage. Segment reuse binds this +// per-bucket suffix to an earlier durable proof from the same process epoch. +pub(super) static DIRTY_USAGE_PRODUCER_IDENTITIES: LazyLock> = + LazyLock::new(|| StdMutex::new(BTreeMap::new())); +static DIRTY_USAGE_CLEAR_OBSERVER: LazyLock>> = + LazyLock::new(|| StdRwLock::new(None)); +static DIRTY_USAGE_MUTATION_OBSERVER: LazyLock>> = + LazyLock::new(|| StdRwLock::new(None)); +pub(super) static DIRTY_USAGE_PRODUCER_COVERAGE: AtomicU64 = AtomicU64::new(0); pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock = LazyLock::new(Notify::new); pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock = LazyLock::new(|| format!("{:032x}", rand::random::())); pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0); pub(super) static SCANNER_MAINTENANCE_NOTIFY: LazyLock = LazyLock::new(Notify::new); +const SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA: u16 = 1; +const SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES: usize = 64 * 1024; +const SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES: usize = 1024; +const SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES: usize = 128; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ScannerDirtyUsageState { @@ -39,6 +54,10 @@ pub struct ScannerDirtyUsageBucket { pub generation: u64, } +pub type ScannerDirtyUsageClearObserver = Arc) + Send + Sync + 'static>; +pub type ScannerDirtyUsageMutationObserver = + Arc; + /// A non-durable optimization hint for a dirty bucket. /// /// A whole-bucket marker always wins over narrow path hints. The scanner never @@ -52,7 +71,97 @@ pub(super) enum DirtyUsageBucketScope { pub(super) type DirtyUsageBucketScopes = HashMap; -const MAX_DIRTY_USAGE_TOP_LEVEL_ENTRIES_PER_BUCKET: usize = 128; +pub(super) const MAX_DIRTY_USAGE_TOP_LEVEL_ENTRIES_PER_BUCKET: usize = 128; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct DirtyUsageProducerIdentityState { + first_generation: u64, + last_generation: u64, + fully_identified: bool, + durable_replayed: bool, +} + +pub(super) type DirtyUsageProducerIdentities = BTreeMap; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct DirtyUsageProducerEvidence { + pub(super) producer_identity_coverage_complete: bool, + pub(super) durable_producer_identity: bool, + pub(super) durable_dirty_producer_journal: bool, + pub(super) restart_gap_absent: bool, + pub(super) generation_window_bound: bool, + pub(super) generation_start: u64, + pub(super) generation_end: u64, +} + +impl DirtyUsageProducerEvidence { + pub(super) fn segment_invalidation_proof(self) -> Option { + (self.generation_window_bound && self.producer_identity_coverage_complete).then(|| { + crate::DataUsageSegmentInvalidationProof { + process_epoch: scanner_activity_epoch().to_string(), + generation_start: self.generation_start, + generation_end: self.generation_end, + producer_identity_coverage_complete: true, + cold_zero_walk_oracle: false, + } + }) + } +} + +pub fn set_scanner_dirty_usage_clear_observer( + observer: Option, +) -> Option { + let mut slot = DIRTY_USAGE_CLEAR_OBSERVER + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + std::mem::replace(&mut *slot, observer) +} + +pub fn set_scanner_dirty_usage_mutation_observer( + observer: Option, +) -> Option { + let mut slot = DIRTY_USAGE_MUTATION_OBSERVER + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + std::mem::replace(&mut *slot, observer) +} + +fn notify_dirty_usage_clear(cleared: Vec) { + if cleared.is_empty() { + return; + } + let observer = DIRTY_USAGE_CLEAR_OBSERVER + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + if let Some(observer) = observer { + observer(cleared); + } +} + +fn notify_dirty_usage_mutation( + bucket: &str, + object: &str, + producers: &[crate::segment_invalidation::SegmentInvalidationProducerIdentity], +) { + if bucket.is_empty() { + return; + } + let observer = DIRTY_USAGE_MUTATION_OBSERVER + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + let Some(observer) = observer else { + return; + }; + if producers.is_empty() { + observer(bucket, object, crate::segment_invalidation::SegmentInvalidationProducerIdentity::Unknown); + } else { + for producer in producers { + observer(bucket, object, *producer); + } + } +} /// A point-in-time view of the local dirty bucket generations. /// @@ -77,6 +186,185 @@ pub enum ScannerDirtyUsageAckError { IncarnationUnavailable, } +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum ScannerDurableDirtyUsageReplayError { + #[error("scanner durable dirty usage replay record exceeds size limit")] + ByteLimit, + #[error("scanner durable dirty usage replay record cannot be decoded")] + InvalidJson, + #[error("scanner durable dirty usage replay schema is unsupported")] + UnsupportedSchema, + #[error("scanner durable dirty usage replay record is empty or over entry limit")] + EntryLimit, + #[error("scanner durable dirty usage replay record contains invalid generation")] + InvalidGeneration, + #[error("scanner durable dirty usage replay record contains an invalid bucket or scope")] + InvalidScope, + #[error("scanner durable dirty usage replay record contains an invalid producer identity")] + InvalidProducer, + #[error("scanner durable dirty usage replay record contains duplicate buckets")] + DuplicateBucket, + #[error("scanner durable dirty usage replay record belongs to a different cache key format")] + MixedVersion, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScannerDurableDirtyUsageReplayRecord { + pub schema: u16, + pub cache_key_format: u16, + pub writer_epoch: String, + pub entries: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScannerDurableDirtyUsageReplayEntry { + pub bucket: String, + pub generation: u64, + pub scope: ScannerDurableDirtyUsageReplayScope, + pub producers: BTreeSet, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ScannerDurableDirtyUsageReplayScope { + WholeBucket, + TopLevelEntries { entries: BTreeSet }, +} + +pub fn encode_durable_dirty_usage_producer_replay_record( + entries: Vec, +) -> std::result::Result, ScannerDurableDirtyUsageReplayError> { + let record = ScannerDurableDirtyUsageReplayRecord { + schema: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA, + cache_key_format: crate::DATA_USAGE_CACHE_KEY_FORMAT, + writer_epoch: scanner_activity_epoch().to_string(), + entries, + }; + validate_durable_dirty_usage_replay_record(&record)?; + serde_json::to_vec(&record).map_err(|_| ScannerDurableDirtyUsageReplayError::InvalidJson) +} + +pub fn replay_durable_dirty_usage_producer_record( + bytes: &[u8], +) -> std::result::Result { + if bytes.len() > SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES { + return Err(ScannerDurableDirtyUsageReplayError::ByteLimit); + } + let record: ScannerDurableDirtyUsageReplayRecord = + serde_json::from_slice(bytes).map_err(|_| ScannerDurableDirtyUsageReplayError::InvalidJson)?; + let replay = validate_durable_dirty_usage_replay_record(&record)?; + let max_generation = replay.iter().map(|entry| entry.generation).max().unwrap_or(0); + set_dirty_usage_generation_floor(max_generation)?; + let pending = { + let mut dirty_buckets = dirty_usage_buckets(); + let mut dirty_scopes = dirty_usage_bucket_scopes(); + let mut producer_identities = dirty_usage_producer_identities(); + let mut coverage = 0_u64; + for entry in replay { + dirty_buckets.insert(entry.bucket.clone(), entry.generation); + dirty_scopes.insert(entry.bucket.clone(), entry.scope); + coverage |= entry.coverage; + producer_identities.insert( + entry.bucket, + DirtyUsageProducerIdentityState { + first_generation: entry.generation, + last_generation: entry.generation, + fully_identified: true, + durable_replayed: true, + }, + ); + } + DIRTY_USAGE_PRODUCER_COVERAGE.fetch_or(coverage, Ordering::AcqRel); + dirty_buckets.len() + }; + global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending)); + DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); + Ok(scanner_dirty_usage_state()) +} + +#[derive(Debug)] +struct ValidatedDurableDirtyUsageReplayEntry { + bucket: String, + generation: u64, + scope: DirtyUsageBucketScope, + coverage: u64, +} + +fn validate_durable_dirty_usage_replay_record( + record: &ScannerDurableDirtyUsageReplayRecord, +) -> std::result::Result, ScannerDurableDirtyUsageReplayError> { + if record.schema != SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA { + return Err(ScannerDurableDirtyUsageReplayError::UnsupportedSchema); + } + if record.cache_key_format != crate::DATA_USAGE_CACHE_KEY_FORMAT { + return Err(ScannerDurableDirtyUsageReplayError::MixedVersion); + } + if record.writer_epoch.is_empty() || record.writer_epoch.contains('\0') { + return Err(ScannerDurableDirtyUsageReplayError::InvalidScope); + } + if record.entries.is_empty() || record.entries.len() > SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES { + return Err(ScannerDurableDirtyUsageReplayError::EntryLimit); + } + let mut buckets = HashSet::with_capacity(record.entries.len()); + let mut replay = Vec::with_capacity(record.entries.len()); + for entry in &record.entries { + if !valid_bucket_for_durable_dirty_usage_replay(&entry.bucket) { + return Err(ScannerDurableDirtyUsageReplayError::InvalidScope); + } + if !buckets.insert(entry.bucket.clone()) { + return Err(ScannerDurableDirtyUsageReplayError::DuplicateBucket); + } + if entry.generation == 0 || entry.generation == u64::MAX { + return Err(ScannerDurableDirtyUsageReplayError::InvalidGeneration); + } + let scope = validate_durable_dirty_usage_replay_scope(&entry.scope)?; + let mut coverage = 0_u64; + for producer in &entry.producers { + let Some(bit) = producer.production_coverage_bit() else { + return Err(ScannerDurableDirtyUsageReplayError::InvalidProducer); + }; + coverage |= bit; + } + if coverage == 0 { + return Err(ScannerDurableDirtyUsageReplayError::InvalidProducer); + } + replay.push(ValidatedDurableDirtyUsageReplayEntry { + bucket: entry.bucket.clone(), + generation: entry.generation, + scope, + coverage, + }); + } + Ok(replay) +} + +fn valid_bucket_for_durable_dirty_usage_replay(bucket: &str) -> bool { + !bucket.is_empty() && !bucket.contains(['/', '\\', '\0']) && bucket != "." && bucket != ".." +} + +fn validate_durable_dirty_usage_replay_scope( + scope: &ScannerDurableDirtyUsageReplayScope, +) -> std::result::Result { + match scope { + ScannerDurableDirtyUsageReplayScope::WholeBucket => Ok(DirtyUsageBucketScope::WholeBucket), + ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } => { + if entries.is_empty() || entries.len() > SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES { + return Err(ScannerDurableDirtyUsageReplayError::InvalidScope); + } + let mut validated = HashSet::with_capacity(entries.len()); + for entry in entries { + if dirty_usage_top_level_entry(entry).as_deref() != Some(entry.as_str()) { + return Err(ScannerDurableDirtyUsageReplayError::InvalidScope); + } + validated.insert(entry.clone()); + } + Ok(DirtyUsageBucketScope::TopLevelEntries(validated)) + } + } +} + /// A scoped ACK requires storage-owned lifecycle and incarnation fences. /// Callers must only send ACKs backed by durable per-bucket publication. pub fn acknowledge_scoped_dirty_usage( @@ -86,9 +374,10 @@ pub fn acknowledge_scoped_dirty_usage( ) -> std::result::Result { // Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map. // No await or storage operation occurs while the dirty map is locked. - let (cleared, pending) = { + let (cleared, pending, cleared_buckets) = { let mut dirty = dirty_usage_buckets(); let mut dirty_scopes = dirty_usage_bucket_scopes(); + let mut producer_identities = dirty_usage_producer_identities(); let checked = entries .iter() .map(|(guard, generation)| { @@ -98,6 +387,18 @@ pub fn acknowledge_scoped_dirty_usage( .map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable) }) .collect::, _>>()?; + let cleared_buckets = if probe_only { + Vec::new() + } else { + checked + .iter() + .filter(|(bucket, generation)| dirty.get(*bucket) == Some(generation)) + .map(|(bucket, generation)| ScannerDirtyUsageBucket { + bucket: (*bucket).to_string(), + generation: *generation, + }) + .collect::>() + }; let cleared = apply_scoped_dirty_usage_ack( instance_id, scanner_activity_epoch(), @@ -108,10 +409,12 @@ pub fn acknowledge_scoped_dirty_usage( probe_only, )?; if cleared > 0 { + producer_identities.retain(|bucket, _| dirty.contains_key(bucket)); advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); } - (cleared, dirty.len()) + (cleared, dirty.len(), cleared_buckets) }; + notify_dirty_usage_clear(cleared_buckets); if !probe_only { global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending)); } @@ -153,6 +456,8 @@ fn apply_scoped_dirty_usage_ack( #[cfg(test)] mod scoped_dirty_usage_tests { use super::*; + use crate::segment_invalidation::SegmentInvalidationProducerIdentity; + use serial_test::serial; #[test] fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() { @@ -213,6 +518,233 @@ mod scoped_dirty_usage_tests { assert_eq!(scopes, original_scopes); } } + + #[test] + #[serial] + fn dirty_usage_tracks_known_segment_producer_identities_without_authorizing_unknown_sources() { + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_object_from_producer("photos", "hot/object", SegmentInvalidationProducerIdentity::PutObject); + record_dirty_usage_object_from_producer("photos", "archive/object", SegmentInvalidationProducerIdentity::DeleteObject); + record_dirty_usage_bucket_from_producer("photos", SegmentInvalidationProducerIdentity::Unknown); + record_dirty_usage_bucket_from_producers( + "photos", + [ + SegmentInvalidationProducerIdentity::DeleteMarker, + SegmentInvalidationProducerIdentity::AbortMultipartUpload, + ], + ); + + assert_eq!( + dirty_usage_producer_identities_for_tests(), + BTreeSet::from([ + SegmentInvalidationProducerIdentity::PutObject, + SegmentInvalidationProducerIdentity::DeleteObject, + SegmentInvalidationProducerIdentity::DeleteMarker, + SegmentInvalidationProducerIdentity::AbortMultipartUpload + ]) + ); + assert_eq!( + dirty_usage_bucket_scopes_for_tests().get("photos"), + Some(&DirtyUsageBucketScope::WholeBucket), + "an unknown producer keeps the bucket dirty but must not count as producer coverage" + ); + let snapshot = snapshot_dirty_usage_buckets( + &[BucketInfo { + name: "photos".to_string(), + created: None, + deleted: None, + versioning: false, + object_locking: false, + }], + dirty_usage_generation(), + ); + let evidence = dirty_usage_producer_evidence(&snapshot); + assert!(!evidence.producer_identity_coverage_complete); + assert!(!evidence.generation_window_bound); + + clear_dirty_usage_buckets_for_tests(); + assert!(dirty_usage_producer_identities_for_tests().is_empty()); + } + + #[test] + #[serial] + fn durable_dirty_usage_replay_restores_producer_identity_authority() { + clear_dirty_usage_buckets_for_tests(); + let bytes = encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry { + bucket: "photos".to_string(), + generation: 7, + scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: BTreeSet::from(["2026".to_string(), "archive".to_string()]), + }, + producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(), + }]) + .expect("durable replay record should encode"); + + let state = replay_durable_dirty_usage_producer_record(&bytes).expect("durable replay should be accepted"); + assert!(state.pending); + assert!(state.generation >= 7); + assert_eq!( + dirty_usage_bucket_scopes_for_tests().get("photos"), + Some(&DirtyUsageBucketScope::TopLevelEntries(HashSet::from([ + "2026".to_string(), + "archive".to_string() + ]))) + ); + + let snapshot = snapshot_dirty_usage_buckets( + &[BucketInfo { + name: "photos".to_string(), + created: None, + deleted: None, + versioning: false, + object_locking: false, + }], + dirty_usage_generation(), + ); + let evidence = dirty_usage_producer_evidence(&snapshot); + assert!(evidence.producer_identity_coverage_complete); + assert!(evidence.durable_producer_identity); + assert!(evidence.durable_dirty_producer_journal); + assert!(evidence.restart_gap_absent); + assert_eq!(evidence.generation_start, 7); + assert_eq!(evidence.generation_end, 7); + + record_dirty_usage_object_from_producer("photos", "new/object", SegmentInvalidationProducerIdentity::PutObject); + let changed = snapshot_dirty_usage_buckets( + &[BucketInfo { + name: "photos".to_string(), + created: None, + deleted: None, + versioning: false, + object_locking: false, + }], + dirty_usage_generation(), + ); + let changed_evidence = dirty_usage_producer_evidence(&changed); + assert!(!changed_evidence.durable_producer_identity); + assert!(!changed_evidence.durable_dirty_producer_journal); + assert!(!changed_evidence.restart_gap_absent); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn dirty_usage_mutation_observer_tracks_typed_and_conservative_events() { + use std::sync::{Arc, Mutex}; + + clear_dirty_usage_buckets_for_tests(); + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_clone = observed.clone(); + let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, producer| { + observed_clone.lock().expect("observer lock should not be poisoned").push(( + bucket.to_string(), + object.to_string(), + producer, + )); + }))); + + record_dirty_usage_object_from_producer("photos", "2026/image.jpg", SegmentInvalidationProducerIdentity::PutObject); + record_dirty_usage_bucket("archive"); + set_scanner_dirty_usage_mutation_observer(previous); + + assert_eq!( + *observed.lock().expect("observer lock should not be poisoned"), + vec![ + ( + "photos".to_string(), + "2026/image.jpg".to_string(), + SegmentInvalidationProducerIdentity::PutObject, + ), + ("archive".to_string(), String::new(), SegmentInvalidationProducerIdentity::Unknown,), + ] + ); + clear_dirty_usage_buckets_for_tests(); + } + + #[test] + #[serial] + fn durable_dirty_usage_replay_rejects_invalid_records_without_partial_state() { + let valid_entry = ScannerDurableDirtyUsageReplayEntry { + bucket: "photos".to_string(), + generation: 7, + scope: ScannerDurableDirtyUsageReplayScope::WholeBucket, + producers: BTreeSet::from([SegmentInvalidationProducerIdentity::PutObject]), + }; + let valid_record = || ScannerDurableDirtyUsageReplayRecord { + schema: SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA, + cache_key_format: crate::DATA_USAGE_CACHE_KEY_FORMAT, + writer_epoch: "writer".to_string(), + entries: vec![valid_entry.clone()], + }; + + let invalid_cases = [ + { + let mut record = valid_record(); + record.schema = SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA + 1; + ( + serde_json::to_vec(&record).expect("invalid schema record should encode"), + ScannerDurableDirtyUsageReplayError::UnsupportedSchema, + ) + }, + { + let mut record = valid_record(); + record.cache_key_format = crate::DATA_USAGE_CACHE_KEY_FORMAT + 1; + ( + serde_json::to_vec(&record).expect("mixed key format record should encode"), + ScannerDurableDirtyUsageReplayError::MixedVersion, + ) + }, + { + let mut record = valid_record(); + record.entries[0].producers = BTreeSet::from([SegmentInvalidationProducerIdentity::Unknown]); + ( + serde_json::to_vec(&record).expect("unknown producer record should encode"), + ScannerDurableDirtyUsageReplayError::InvalidProducer, + ) + }, + { + let mut record = valid_record(); + record.entries.push(valid_entry.clone()); + ( + serde_json::to_vec(&record).expect("duplicate bucket record should encode"), + ScannerDurableDirtyUsageReplayError::DuplicateBucket, + ) + }, + { + let mut record = valid_record(); + record.entries[0].scope = ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: BTreeSet::from(["bad/child".to_string()]), + }; + ( + serde_json::to_vec(&record).expect("invalid scope record should encode"), + ScannerDurableDirtyUsageReplayError::InvalidScope, + ) + }, + { + let mut record = valid_record(); + record.entries[0].generation = u64::MAX; + ( + serde_json::to_vec(&record).expect("invalid generation record should encode"), + ScannerDurableDirtyUsageReplayError::InvalidGeneration, + ) + }, + ]; + + for (bytes, expected_error) in invalid_cases { + clear_dirty_usage_buckets_for_tests(); + assert_eq!(replay_durable_dirty_usage_producer_record(&bytes), Err(expected_error)); + assert!(dirty_usage_buckets_for_tests().is_empty()); + assert!(dirty_usage_bucket_scopes_for_tests().is_empty()); + } + + clear_dirty_usage_buckets_for_tests(); + let oversized = vec![b' '; SCANNER_DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES + 1]; + assert_eq!( + replay_durable_dirty_usage_producer_record(&oversized), + Err(ScannerDurableDirtyUsageReplayError::ByteLimit) + ); + assert!(dirty_usage_buckets_for_tests().is_empty()); + } } pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> { @@ -225,6 +757,12 @@ fn dirty_usage_bucket_scopes() -> MutexGuard<'static, DirtyUsageBucketScopes> { .unwrap_or_else(|poisoned| poisoned.into_inner()) } +fn dirty_usage_producer_identities() -> MutexGuard<'static, DirtyUsageProducerIdentities> { + DIRTY_USAGE_PRODUCER_IDENTITIES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + pub(super) fn usize_to_u64_saturated(value: usize) -> u64 { u64::try_from(value).unwrap_or(u64::MAX) } @@ -235,17 +773,66 @@ pub(super) fn advance_generation(generation: &AtomicU64) -> u64 { .map_or_else(|current| current, |previous| previous.saturating_add(1)) } +fn set_dirty_usage_generation_floor(floor: u64) -> std::result::Result<(), ScannerDurableDirtyUsageReplayError> { + if floor == 0 || floor == u64::MAX { + return Err(ScannerDurableDirtyUsageReplayError::InvalidGeneration); + } + DIRTY_USAGE_BUCKET_GENERATION + .try_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current != u64::MAX).then_some(current.max(floor)) + }) + .map(|_| ()) + .map_err(|_| ScannerDurableDirtyUsageReplayError::InvalidGeneration) +} + pub fn record_dirty_usage_bucket(bucket: &str) { if bucket.is_empty() { return; } + record_dirty_usage_bucket_inner(bucket, std::iter::empty()); +} + +pub fn record_dirty_usage_bucket_from_producer( + bucket: &str, + producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity, +) { + if bucket.is_empty() { + return; + } + + record_dirty_usage_bucket_inner(bucket, [producer]); +} + +pub fn record_dirty_usage_bucket_from_producers(bucket: &str, producers: I) +where + I: IntoIterator, +{ + if bucket.is_empty() { + return; + } + + record_dirty_usage_bucket_inner(bucket, producers); +} + +fn record_dirty_usage_bucket_inner(bucket: &str, producers: I) +where + I: IntoIterator, +{ + let producers = producers.into_iter().collect::>(); let pending_buckets = { let mut dirty_buckets = dirty_usage_buckets(); let mut dirty_scopes = dirty_usage_bucket_scopes(); + let mut producer_identities = dirty_usage_producer_identities(); let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); dirty_buckets.insert(bucket.to_string(), generation); dirty_scopes.insert(bucket.to_string(), DirtyUsageBucketScope::WholeBucket); + record_segment_invalidation_producer_identities_for_generation( + &mut producer_identities, + bucket, + generation, + producers.iter().copied(), + ); dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets)); @@ -253,6 +840,7 @@ pub fn record_dirty_usage_bucket(bucket: &str) { // admin/console consumers never ride the full TTL after a change // (rustfs/backlog#1872). crate::prefix_usage::invalidate_prefix_usage_cache(bucket); + notify_dirty_usage_mutation(bucket, "", &producers); DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); } @@ -263,8 +851,28 @@ pub fn record_dirty_usage_bucket(bucket: &str) { /// local: after restart or any unverified distributed path the scanner falls /// back to its ordinary bucket scan. pub fn record_dirty_usage_object(bucket: &str, object: &str) { + record_dirty_usage_object_inner(bucket, object, std::iter::empty()); +} + +pub fn record_dirty_usage_object_from_producer( + bucket: &str, + object: &str, + producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity, +) { + if bucket.is_empty() { + return; + } + + record_dirty_usage_object_inner(bucket, object, [producer]); +} + +fn record_dirty_usage_object_inner(bucket: &str, object: &str, producers: I) +where + I: IntoIterator, +{ + let producers = producers.into_iter().collect::>(); let Some(top_level_entry) = dirty_usage_top_level_entry(object) else { - record_dirty_usage_bucket(bucket); + record_dirty_usage_bucket_inner(bucket, producers); return; }; if bucket.is_empty() { @@ -274,6 +882,7 @@ pub fn record_dirty_usage_object(bucket: &str, object: &str) { let pending_buckets = { let mut dirty_buckets = dirty_usage_buckets(); let mut dirty_scopes = dirty_usage_bucket_scopes(); + let mut producer_identities = dirty_usage_producer_identities(); let generation = advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); dirty_buckets.insert(bucket.to_string(), generation); let scope = dirty_scopes @@ -289,13 +898,69 @@ pub fn record_dirty_usage_object(bucket: &str, object: &str) { if overflowed { *scope = DirtyUsageBucketScope::WholeBucket; } + record_segment_invalidation_producer_identities_for_generation( + &mut producer_identities, + bucket, + generation, + producers.iter().copied(), + ); dirty_buckets.len() }; global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets)); crate::prefix_usage::invalidate_prefix_usage_cache(bucket); + notify_dirty_usage_mutation(bucket, object, &producers); DIRTY_USAGE_BUCKET_NOTIFY.notify_one(); } +fn record_segment_invalidation_producer_identities_for_generation( + identities: &mut DirtyUsageProducerIdentities, + bucket: &str, + generation: u64, + producers: I, +) where + I: IntoIterator, +{ + let mut event_coverage = 0_u64; + let mut fully_identified = true; + for producer in producers { + if let Some(coverage_bit) = producer.production_coverage_bit() { + event_coverage |= coverage_bit; + } else { + fully_identified = false; + } + } + fully_identified &= event_coverage != 0; + DIRTY_USAGE_PRODUCER_COVERAGE.fetch_or(event_coverage, Ordering::AcqRel); + + match identities.get_mut(bucket) { + Some(state) => { + state.last_generation = state.last_generation.max(generation); + state.fully_identified &= fully_identified; + state.durable_replayed = false; + } + None => { + identities.insert( + bucket.to_string(), + DirtyUsageProducerIdentityState { + first_generation: generation, + last_generation: generation, + fully_identified, + durable_replayed: false, + }, + ); + } + } +} + +#[cfg(test)] +fn dirty_usage_producer_identities_for_tests() -> BTreeSet { + let coverage = DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire); + crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION + .into_iter() + .filter(|identity| identity.production_coverage_bit().is_some_and(|bit| coverage & bit != 0)) + .collect() +} + fn dirty_usage_top_level_entry(object: &str) -> Option { let (top_level_entry, _) = object.split_once('/').unwrap_or((object, "")); (!top_level_entry.is_empty() @@ -373,23 +1038,34 @@ pub fn acknowledge_dirty_usage_generation( return Err(ScannerDirtyUsageAckError::ProcessChanged); } - let (cleared_buckets, pending_buckets) = { + let (cleared_buckets, pending_buckets, cleared) = { let mut dirty_buckets = dirty_usage_buckets(); let mut dirty_scopes = dirty_usage_bucket_scopes(); + let mut producer_identities = dirty_usage_producer_identities(); let current_generation = DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire); if generation == 0 || generation == u64::MAX || current_generation == u64::MAX || generation > current_generation { return Err(ScannerDirtyUsageAckError::InvalidGeneration); } let before = dirty_buckets.len(); + let cleared = dirty_buckets + .iter() + .filter(|(_, dirty_generation)| **dirty_generation <= generation) + .map(|(bucket, dirty_generation)| ScannerDirtyUsageBucket { + bucket: bucket.clone(), + generation: *dirty_generation, + }) + .collect::>(); dirty_buckets.retain(|_, dirty_generation| *dirty_generation > generation); dirty_scopes.retain(|bucket, _| dirty_buckets.contains_key(bucket)); + producer_identities.retain(|bucket, _| dirty_buckets.contains_key(bucket)); let cleared_buckets = before.saturating_sub(dirty_buckets.len()); if cleared_buckets > 0 { advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); } - (cleared_buckets, dirty_buckets.len()) + (cleared_buckets, dirty_buckets.len(), cleared) }; + notify_dirty_usage_clear(cleared); global_metrics() .record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets)); Ok(()) @@ -404,14 +1080,23 @@ pub fn clear_dirty_usage_bucket(bucket: &str) { return; } - let pending_buckets = { + let (pending_buckets, cleared) = { let mut dirty_buckets = dirty_usage_buckets(); let mut dirty_scopes = dirty_usage_bucket_scopes(); - dirty_buckets.remove(bucket); + let mut producer_identities = dirty_usage_producer_identities(); + let cleared = dirty_buckets.remove(bucket).map(|generation| ScannerDirtyUsageBucket { + bucket: bucket.to_string(), + generation, + }); dirty_scopes.remove(bucket); + producer_identities.remove(bucket); + DIRTY_USAGE_PRODUCER_COVERAGE.store(0, Ordering::Release); advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); - dirty_buckets.len() + (dirty_buckets.len(), cleared) }; + if let Some(cleared) = cleared { + notify_dirty_usage_clear(vec![cleared]); + } global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets)); } @@ -467,22 +1152,30 @@ pub(crate) async fn dirty_usage_bucket_notified() { } pub(super) fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) { - let (cleared_buckets, pending_buckets) = { + let (cleared_buckets, pending_buckets, cleared) = { let mut dirty_buckets = dirty_usage_buckets(); let mut dirty_scopes = dirty_usage_bucket_scopes(); + let mut producer_identities = dirty_usage_producer_identities(); let mut cleared_buckets = 0usize; + let mut cleared = Vec::new(); for (bucket, generation) in snapshot { if dirty_buckets.get(bucket).is_some_and(|current| current == generation) { dirty_buckets.remove(bucket); dirty_scopes.remove(bucket); cleared_buckets += 1; + cleared.push(ScannerDirtyUsageBucket { + bucket: bucket.clone(), + generation: *generation, + }); } } if cleared_buckets > 0 { + producer_identities.retain(|bucket, _| dirty_buckets.contains_key(bucket)); advance_generation(&DIRTY_USAGE_BUCKET_GENERATION); } - (cleared_buckets, dirty_buckets.len()) + (cleared_buckets, dirty_buckets.len(), cleared) }; + notify_dirty_usage_clear(cleared); global_metrics() .record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets)); } @@ -568,6 +1261,46 @@ pub(super) fn dirty_usage_snapshot_status(snapshot: &DirtyUsageSnapshot) -> Dirt } } +pub(super) fn dirty_usage_producer_evidence(snapshot: &DirtyUsageSnapshot) -> DirtyUsageProducerEvidence { + let snapshot_current = dirty_usage_snapshot_status(snapshot) == DirtyUsageSnapshotStatus::Current + && snapshot.generation != 0 + && snapshot.generation != u64::MAX + && !snapshot.buckets.is_empty(); + let producer_identities = dirty_usage_producer_identities(); + let mut generation_start = u64::MAX; + let mut generation_end = 0; + let producer_identity_coverage_complete = snapshot_current + && snapshot.buckets.iter().all(|(bucket, generation)| { + producer_identities.get(bucket).is_some_and(|state| { + generation_start = generation_start.min(state.first_generation); + generation_end = generation_end.max(state.last_generation); + state.fully_identified && state.last_generation == *generation + }) + }) + && DIRTY_USAGE_PRODUCER_COVERAGE.load(Ordering::Acquire) + & crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK + == crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK; + let durable_dirty_producer_journal = producer_identity_coverage_complete + && snapshot + .buckets + .keys() + .all(|bucket| producer_identities.get(bucket).is_some_and(|state| state.durable_replayed)); + let generation_window_bound = producer_identity_coverage_complete + && generation_start != u64::MAX + && generation_end >= generation_start + && generation_end <= snapshot.generation; + + DirtyUsageProducerEvidence { + producer_identity_coverage_complete, + durable_producer_identity: durable_dirty_producer_journal, + durable_dirty_producer_journal, + restart_gap_absent: durable_dirty_producer_journal, + generation_window_bound, + generation_start: if generation_window_bound { generation_start } else { 0 }, + generation_end: if generation_window_bound { generation_end } else { 0 }, + } +} + #[cfg(test)] pub(super) fn dirty_usage_bucket_count() -> usize { dirty_usage_buckets().len() @@ -577,6 +1310,8 @@ pub(super) fn dirty_usage_bucket_count() -> usize { pub(crate) fn clear_dirty_usage_buckets_for_tests() { dirty_usage_buckets().clear(); dirty_usage_bucket_scopes().clear(); + dirty_usage_producer_identities().clear(); + DIRTY_USAGE_PRODUCER_COVERAGE.store(0, Ordering::Release); } #[cfg(test)] diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index 2cb970d8e..d99e889ad 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -26,6 +26,21 @@ pub(super) struct ScannerSetCacheGeneration { pub(super) struct PreparedScopedSetScan { pub(super) buckets: Vec, pub(super) cache: DataUsageCache, + pub(super) cold_bucket_reuse_proof: Option, +} + +pub(super) struct ScopedColdBucketReuseProof { + pub(super) baseline_scan_plan_digest: DataUsageScanPlanDigest, + pub(super) source: DataUsageCacheSource, + pub(super) bucket_incarnations: HashMap, +} + +impl ScopedColdBucketReuseProof { + fn authorizes(&self, source: DataUsageCacheSource, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> bool { + self.source == source + && self.baseline_scan_plan_digest == baseline_scan_plan_digest + && !self.bucket_incarnations.is_empty() + } } pub(super) fn prepare_scoped_set_scan( @@ -51,10 +66,16 @@ pub(super) fn prepare_scoped_set_scan( || old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest) || old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT || !old_cache.has_complete_root_inventory(&old_cache.find(DATA_USAGE_ROOT)?.children) - || !unselected_bucket_incarnations_match(old_cache, all_buckets, selected_buckets, current_bucket_incarnations) { return None; } + let unselected_bucket_incarnations = + unselected_bucket_incarnation_bindings(old_cache, all_buckets, selected_buckets, current_bucket_incarnations)?; + let cold_bucket_reuse_proof = (!unselected_bucket_incarnations.is_empty()).then_some(ScopedColdBucketReuseProof { + baseline_scan_plan_digest, + source: generation.source, + bucket_incarnations: unselected_bucket_incarnations, + }); let mut cache = DataUsageCache { info: DataUsageCacheInfo { @@ -100,35 +121,41 @@ pub(super) fn prepare_scoped_set_scan( .cloned() .collect(), cache, + cold_bucket_reuse_proof, }) } -fn unselected_bucket_incarnations_match( +fn unselected_bucket_incarnation_bindings( old_cache: &DataUsageCache, all_buckets: &[BucketInfo], selected_buckets: &HashSet, current_bucket_incarnations: Option<&HashMap>, -) -> bool { - let Some(current_bucket_incarnations) = current_bucket_incarnations else { - return all_buckets.iter().all(|bucket| selected_buckets.contains(&bucket.name)); - }; - all_buckets +) -> Option> { + let mut unselected_buckets = all_buckets .iter() .filter(|bucket| !selected_buckets.contains(&bucket.name)) - .all(|bucket| { - let Some(current) = current_bucket_incarnations - .get(&bucket.name) - .filter(|incarnation| !incarnation.is_nil()) - else { - return false; - }; - old_cache + .peekable(); + let Some(current_bucket_incarnations) = current_bucket_incarnations else { + return unselected_buckets.peek().is_none().then(HashMap::new); + }; + let mut bound_incarnations = HashMap::new(); + for bucket in unselected_buckets { + let current = current_bucket_incarnations + .get(&bucket.name) + .filter(|incarnation| !incarnation.is_nil())?; + if old_cache.find(&bucket.name).is_none() + || old_cache .info .scan_bucket_incarnations .get(&bucket.name) .filter(|cached| !cached.is_nil()) - == Some(current) - }) + != Some(current) + { + return None; + } + bound_incarnations.insert(bucket.name.clone(), *current); + } + Some(bound_incarnations) } async fn scanner_current_bucket_incarnations(set: &SetDisks, all_buckets: &[BucketInfo]) -> Option> { @@ -172,6 +199,8 @@ impl ScannerIOCache for SetDisks { bucket_failures, pending_maintenance_work, cache_cycle_floor, + cold_zero_walk_reuse_observed, + segment_invalidation_proof, } = scan_plan; let scan_plan_digest = scanner_bucket_work_digest(scan_plan_digest, scan_mode, requires_full_scan); let bucket_work_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan); @@ -219,6 +248,15 @@ impl ScannerIOCache for SetDisks { }, current_bucket_incarnations.as_ref(), ); + let cold_zero_walk_reuse_candidate = scoped_scan.as_ref().is_some_and(|prepared| { + old_cache.info.next_cycle < want_cycle + && scope.baseline_scan_plan_digest.is_some_and(|baseline_scan_plan_digest| { + prepared + .cold_bucket_reuse_proof + .as_ref() + .is_some_and(|proof| proof.authorizes(source, baseline_scan_plan_digest)) + }) + }); let mut scoped_cache = scoped_scan.map(|mut prepared| { buckets = prepared.buckets; prepared.cache.info.scan_coverage_digest = Some(bucket_coverage_digest); @@ -226,6 +264,8 @@ impl ScannerIOCache for SetDisks { }); if buckets.is_empty() { let now = SystemTime::now(); + let completed_segment_invalidation_proof = + scanner_completed_set_segment_invalidation_proof(&segment_invalidation_proof, cold_zero_walk_reuse_candidate); let mut cache = match scoped_cache.take() { Some(cache) => cache, None => { @@ -239,6 +279,7 @@ impl ScannerIOCache for SetDisks { scan_plan_digest: Some(scan_plan_digest), scan_coverage_digest: Some(bucket_coverage_digest), cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + segment_invalidation_proof: completed_segment_invalidation_proof.clone(), scan_bucket_incarnations: current_bucket_incarnations.clone().unwrap_or_default(), ..Default::default() }, @@ -254,11 +295,15 @@ impl ScannerIOCache for SetDisks { cache.info.last_update = Some(now); cache.info.snapshot_complete = true; cache.info.scan_execution_digest = Some(execution_digest); + cache.info.segment_invalidation_proof = completed_segment_invalidation_proof; cache.info.lkg_snapshot_complete = false; cache.info.lkg_next_cycle = None; cache.info.lkg_last_update = None; cache.info.lkg_leader_epoch = None; cache.info.lkg_scan_plan_digest = None; + if cold_zero_walk_reuse_candidate { + cold_zero_walk_reuse_observed.store(true, Ordering::Release); + } if cache.find(DATA_USAGE_ROOT).is_none() { cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default()); } @@ -535,6 +580,7 @@ impl ScannerIOCache for SetDisks { lkg_last_update: old_cache.info.lkg_last_update, lkg_leader_epoch: old_cache.info.lkg_leader_epoch, lkg_scan_plan_digest: old_cache.info.lkg_scan_plan_digest, + segment_invalidation_proof: None, scan_bucket_incarnations: current_bucket_incarnations.clone().unwrap_or_default(), ..Default::default() }, @@ -1455,17 +1501,23 @@ impl ScannerIOCache for SetDisks { let completed_count = completed_bucket_count.load(Ordering::Relaxed); if should_publish_completed_snapshot(completed_count, buckets.len(), budget.budget_elapsed(), ctx.is_cancelled()) { + let completed_segment_invalidation_proof = + scanner_completed_set_segment_invalidation_proof(&segment_invalidation_proof, cold_zero_walk_reuse_candidate); let cache_snapshot = { let mut cache = cache_mutex.lock().await; cache.info.next_cycle = want_cycle; cache.info.last_update.get_or_insert_with(SystemTime::now); cache.info.snapshot_complete = true; cache.info.scan_execution_digest = Some(execution_digest); + cache.info.segment_invalidation_proof = completed_segment_invalidation_proof; cache.info.lkg_snapshot_complete = false; cache.info.lkg_next_cycle = None; cache.info.lkg_last_update = None; cache.info.lkg_leader_epoch = None; cache.info.lkg_scan_plan_digest = None; + if cold_zero_walk_reuse_candidate { + cold_zero_walk_reuse_observed.store(true, Ordering::Release); + } cache.clone() }; let _ = persist_and_publish_cache_snapshot( @@ -1486,6 +1538,7 @@ impl ScannerIOCache for SetDisks { incomplete_scope.info.tier_registry_generation = Some(tier_registry_generation); incomplete_scope.info.source = Some(source); incomplete_scope.info.snapshot_complete = false; + incomplete_scope.info.segment_invalidation_proof = None; incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest); incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT; if let Err(e) = updates.send(incomplete_scope).await { diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 8535468ed..08a177984 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -122,6 +122,7 @@ where let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult { scope, remote_dirty_usage_acknowledgements: Vec::new(), + distributed_segment_invalidation_evidence: None, }; if resolution.requires_full_scan { return default_result(ScannerBucketScanScope::default()); @@ -189,7 +190,7 @@ where }; let remote_resolution = resolve_remote_dirty_usage_scope( resolution.requested_scope, - dirty_buckets, + resolution.dirty_usage_snapshot, remote_dirty_usage, resolution.all_buckets, resolution.baseline_proof, @@ -240,12 +241,17 @@ where return remote_resolution; } + let segment_reuse_activation_preflight = scanner_segment_reuse_activation_preflight_for_baseline( + resolution.dirty_usage_snapshot, + distributed, + resolution.baseline_proof, + ); default_result(scoped_scan_scope_from_dirty_buckets( resolution.requested_scope, dirty_buckets, (!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()), true, - scanner_segment_reuse_activated(), + segment_reuse_activation_preflight.scanner_segment_reuse_activated, resolution.all_buckets, resolution.baseline_proof, )) @@ -408,7 +414,9 @@ where ) .await; let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements; + let distributed_segment_invalidation_evidence = scope_resolution.distributed_segment_invalidation_evidence; let scan_scope = scope_resolution.scope; + let segment_invalidation_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot).segment_invalidation_proof(); #[cfg(test)] if let Some(observer) = resolved_scope_observer { let _ = observer.send(scan_scope.clone()); @@ -465,12 +473,20 @@ where } else { Vec::new() }; + let segment_reuse_activation_preflight = scanner_segment_reuse_activation_preflight_for_cycle( + &dirty_usage_snapshot, + dirty_usage_producer_evidence(&dirty_usage_snapshot), + distributed, + None, + false, + ); return Ok(ScannerCycleResult::new(status, dirty_usage_clear) .with_publication_epoch(publication_epoch) .with_activity_digest(activity_digest) .with_observational_snapshot_published(observational_snapshot_published) .with_remote_publication_lease_targets(remote_publication_lease_targets) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) + .with_segment_reuse_activation_preflight(segment_reuse_activation_preflight) .with_publication_expectation(publication_expectation)); } @@ -495,6 +511,7 @@ where ); let bucket_failures = ScannerBucketFailureState::default(); let pending_maintenance_work = Arc::new(AtomicBool::new(false)); + let cold_zero_walk_reuse_observed = Arc::new(AtomicBool::new(false)); record_set_scan_concurrency_limit(set_scan_limit); debug!( target: "rustfs::scanner::io", @@ -588,6 +605,8 @@ where bucket_failures: bucket_failures.clone(), pending_maintenance_work: pending_maintenance_work.clone(), cache_cycle_floor: cache_cycle_floor.clone(), + cold_zero_walk_reuse_observed: cold_zero_walk_reuse_observed.clone(), + segment_invalidation_proof: segment_invalidation_proof.clone(), }; // Spawn task to run the scanner let scanner_fut = tokio::spawn(async move { @@ -691,6 +710,21 @@ where scan_scope_matches && !partial_buckets.is_empty(), scan_scope_matches && !namespace_not_found_buckets.is_empty(), ); + let cold_zero_walk_oracle = scanner_cycle_cold_zero_walk_oracle( + &scan_scope, + &all_buckets, + completed_all_sets, + scan_scope_matches, + bucket_scan_status, + cold_zero_walk_reuse_observed.load(Ordering::Acquire), + ); + let segment_reuse_activation_preflight = scanner_segment_reuse_activation_preflight_for_cycle( + &dirty_usage_snapshot, + scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &results, &expected_sources), + distributed, + distributed_segment_invalidation_evidence, + cold_zero_walk_oracle, + ); let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results); let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire); let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor); @@ -783,6 +817,8 @@ where .with_observational_snapshot_published(observational_snapshot_published) .with_remote_publication_lease_targets(remote_publication_lease_targets) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) + .with_distributed_segment_invalidation_evidence(distributed_segment_invalidation_evidence) + .with_segment_reuse_activation_preflight(segment_reuse_activation_preflight) .with_failed_dirty_usage(!failed_buckets.is_empty()) .with_pending_maintenance_work(pending_maintenance_work) .with_required_cycle_floor(required_cycle_floor) diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index 6274f27fa..564ea2baf 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::*; -use crate::data_usage_define::{UNKNOWN_TIER, UnknownTierStats, hash_path}; +use crate::data_usage_define::{DataUsageSegmentInvalidationProof, UNKNOWN_TIER, UnknownTierStats, hash_path}; use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage, TierAccountingProof}; const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]); @@ -176,6 +176,27 @@ fn completed_data_usage_info_rejects_duplicate_bucket_inventory() { assert!(completed_data_usage_info_for_test(&[set], &buckets, false, false).is_none()); } +#[test] +fn completed_data_usage_info_carries_segment_invalidation_proof_to_set_state() { + let source = DataUsageCacheSource::new(0, 0); + let proof = DataUsageSegmentInvalidationProof { + process_epoch: "scanner-process".to_string(), + generation_start: 5, + generation_end: 8, + producer_identity_coverage_complete: true, + cold_zero_walk_oracle: true, + }; + let mut set = completed_root_cache("bucket", 2, 10, source); + set.info.segment_invalidation_proof = Some(proof.clone()); + + let (usage, _) = + completed_usage_for_scope(&[set], &HashSet::from([source]), &["bucket".to_string()], &[], true, false, false) + .expect("complete set should publish root usage"); + + assert_eq!(usage.usage_snapshot_set_states.len(), 1); + assert_eq!(usage.usage_snapshot_set_states[0].segment_invalidation_proof, Some(proof)); +} + #[test] fn completed_data_usage_info_rejects_extra_or_detached_bucket_data() { let buckets = vec!["bucket".to_string()]; @@ -350,6 +371,7 @@ fn set_membership_add_remove_uses_generation_and_tombstone() { scan_plan_digest: Some(TEST_PLAN_DIGEST.0), complete: false, tombstone: true, + segment_invalidation_proof: None, }; let encoded = serde_json::to_vec(&state).expect("set state should serialize"); let decoded: DataUsageSnapshotSetState = serde_json::from_slice(&encoded).expect("set state should deserialize"); @@ -371,6 +393,7 @@ fn set_membership_add_remove_uses_generation_and_tombstone() { scan_plan_digest: Some(TEST_PLAN_DIGEST.0), complete: true, tombstone: false, + segment_invalidation_proof: None, }, state, ], diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 01ee44627..de1438fe8 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -37,7 +37,7 @@ use rustfs_concurrency::{ }; use rustfs_filemeta::FileInfo; use serial_test::serial; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use temp_env::with_var; use time::OffsetDateTime; @@ -85,6 +85,441 @@ fn scanner_activity_preflight_defers_a_temporarily_offline_peer() { } } +#[test] +fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() { + let preflight = scanner_segment_reuse_activation_preflight(); + + assert!(preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert!(!scanner_segment_reuse_activated()); + assert_eq!(preflight.proof_inputs, SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS); + assert_eq!(preflight.fail_closed_checks, SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS); + assert_eq!( + preflight.fail_closed_blockers().collect::>(), + vec![ + "missing_producer_identity", + "missing_durable_journal_replay", + "restart_gap", + "generation_gap", + "overflow", + "stale_ack_generation", + "missing_cold_zero_walk_oracle", + "distributed_without_peer_invalidation", + ] + ); +} + +#[test] +fn scanner_segment_reuse_activation_requires_every_preflight_proof() { + let complete_proof = ScannerSegmentReuseActivationProof { + production_activation: true, + producer_identity_coverage_complete: true, + durable_producer_identity: true, + durable_dirty_producer_journal: true, + restart_gap_absent: true, + generation_window_bound: true, + overflow_absent: true, + ack_generation_guard: true, + cold_zero_walk_oracle: true, + distributed_peer_invalidation: true, + }; + + let mut production_disabled = complete_proof; + production_disabled.production_activation = false; + let preflight = scanner_segment_reuse_activation_preflight_from_proof(production_disabled); + assert!(!preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!(preflight.fail_closed_blockers().collect::>(), Vec::<&str>::new()); + + let preflight = scanner_segment_reuse_activation_preflight_from_proof(complete_proof); + assert!(preflight.production_activation); + assert!(preflight.scanner_segment_reuse_activated); + assert_eq!(preflight.fail_closed_blockers().collect::>(), Vec::<&str>::new()); + + let mut missing_identity = complete_proof; + missing_identity.producer_identity_coverage_complete = false; + assert_segment_reuse_activation_blocked_by(missing_identity, "missing_producer_identity"); + + let mut non_durable_identity = complete_proof; + non_durable_identity.durable_producer_identity = false; + assert_segment_reuse_activation_blocked_by(non_durable_identity, "missing_producer_identity"); + + let mut missing_durable_journal = complete_proof; + missing_durable_journal.durable_dirty_producer_journal = false; + assert_segment_reuse_activation_blocked_by(missing_durable_journal, "missing_durable_journal_replay"); + + let mut restart_gap = complete_proof; + restart_gap.restart_gap_absent = false; + assert_segment_reuse_activation_blocked_by(restart_gap, "restart_gap"); + + let mut generation_gap = complete_proof; + generation_gap.generation_window_bound = false; + assert_segment_reuse_activation_blocked_by(generation_gap, "generation_gap"); + + let mut overflow = complete_proof; + overflow.overflow_absent = false; + assert_segment_reuse_activation_blocked_by(overflow, "overflow"); + + let mut stale_ack_generation = complete_proof; + stale_ack_generation.ack_generation_guard = false; + assert_segment_reuse_activation_blocked_by(stale_ack_generation, "stale_ack_generation"); + + let mut missing_cold_oracle = complete_proof; + missing_cold_oracle.cold_zero_walk_oracle = false; + assert_segment_reuse_activation_blocked_by(missing_cold_oracle, "missing_cold_zero_walk_oracle"); + + let mut missing_distributed_invalidation = complete_proof; + missing_distributed_invalidation.distributed_peer_invalidation = false; + assert_segment_reuse_activation_blocked_by(missing_distributed_invalidation, "distributed_without_peer_invalidation"); +} + +#[test] +fn scanner_segment_reuse_activation_preflight_for_cycle_reports_cycle_inputs_without_activation() { + let dirty_usage_snapshot = DirtyUsageSnapshot { + buckets: Arc::new(DirtyUsageBuckets::from([("photos".to_string(), 7)])), + scopes: Arc::new(DirtyUsageBucketScopes::default()), + generation: 7, + covers_all_pending: true, + }; + let distributed_evidence = DistributedSegmentInvalidationEvidence { + invalidation_domain: crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc, + distributed_ec_invalidation: true, + peer_count: 2, + dirty_peer_count: 1, + same_window_remote_proof: true, + all_peers_bound_to_generation_window: true, + }; + + let preflight = scanner_segment_reuse_activation_preflight_for_cycle( + &dirty_usage_snapshot, + complete_process_local_producer_evidence(), + true, + Some(distributed_evidence), + true, + ); + + assert!(preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!( + preflight.fail_closed_blockers().collect::>(), + vec!["missing_producer_identity", "missing_durable_journal_replay", "restart_gap"] + ); +} + +#[test] +fn scanner_segment_reuse_activation_preflight_for_cycle_blocks_unbounded_inputs() { + let dirty_usage_snapshot = DirtyUsageSnapshot { + buckets: Arc::new(DirtyUsageBuckets::default()), + scopes: Arc::new(DirtyUsageBucketScopes::default()), + generation: u64::MAX, + covers_all_pending: false, + }; + + let preflight = scanner_segment_reuse_activation_preflight_for_cycle( + &dirty_usage_snapshot, + DirtyUsageProducerEvidence::default(), + true, + None, + false, + ); + + assert!(preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!( + preflight.fail_closed_blockers().collect::>(), + vec![ + "missing_producer_identity", + "missing_durable_journal_replay", + "restart_gap", + "generation_gap", + "overflow", + "missing_cold_zero_walk_oracle", + "distributed_without_peer_invalidation", + ] + ); +} + +#[test] +fn scanner_segment_reuse_activation_preflight_for_cycle_skips_distributed_blocker_for_local_scan() { + let dirty_usage_snapshot = DirtyUsageSnapshot { + buckets: Arc::new(DirtyUsageBuckets::from([("photos".to_string(), 7)])), + scopes: Arc::new(DirtyUsageBucketScopes::default()), + generation: 7, + covers_all_pending: true, + }; + + let preflight = scanner_segment_reuse_activation_preflight_for_cycle( + &dirty_usage_snapshot, + complete_process_local_producer_evidence(), + false, + None, + true, + ); + + assert!(preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!( + preflight.fail_closed_blockers().collect::>(), + vec!["missing_producer_identity", "missing_durable_journal_replay", "restart_gap"] + ); +} + +#[test] +#[serial] +fn scanner_durable_segment_invalidation_evidence_requires_matching_complete_set_proofs() { + use crate::segment_invalidation::SegmentInvalidationProducerIdentity; + + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket_from_producers("photos", SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION); + let dirty_usage_snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation()); + let process_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot) + .segment_invalidation_proof() + .expect("complete process-local producer coverage should produce proof metadata"); + let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(0, 1)]); + let results = vec![ + complete_set_cache_with_segment_proof(DataUsageCacheSource::new(0, 0), process_proof.clone()), + complete_set_cache_with_segment_proof(DataUsageCacheSource::new(0, 1), process_proof), + ]; + + let durable_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &results, &expected_sources); + + assert!(durable_evidence.producer_identity_coverage_complete); + assert!(durable_evidence.durable_producer_identity); + assert!(!durable_evidence.durable_dirty_producer_journal); + assert!(durable_evidence.restart_gap_absent); + + let mut stale_epoch = results.clone(); + stale_epoch[0] + .info + .segment_invalidation_proof + .as_mut() + .expect("proof fixture should exist") + .process_epoch = "stale-process".to_string(); + let stale_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &stale_epoch, &expected_sources); + assert!(stale_evidence.producer_identity_coverage_complete); + assert!(!stale_evidence.durable_producer_identity); + assert!(!stale_evidence.durable_dirty_producer_journal); + assert!(!stale_evidence.restart_gap_absent); + + record_dirty_usage_bucket("videos"); + let changed_evidence = scanner_durable_segment_invalidation_evidence(&dirty_usage_snapshot, &results, &expected_sources); + assert!(!changed_evidence.producer_identity_coverage_complete); + assert!(!changed_evidence.durable_producer_identity); + assert!(!changed_evidence.durable_dirty_producer_journal); + assert!(!changed_evidence.restart_gap_absent); + clear_dirty_usage_buckets_for_tests(); +} + +#[test] +#[serial] +fn scanner_segment_reuse_activation_replays_cold_durable_baseline() { + use crate::segment_invalidation::SegmentInvalidationProducerIdentity; + + clear_dirty_usage_buckets_for_tests(); + for producer in SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION { + record_dirty_usage_object_from_producer("photos", "2026/object", producer); + } + let replay_generation = dirty_usage_generation(); + replay_durable_dirty_usage_producer_record( + &encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry { + bucket: "photos".to_string(), + generation: replay_generation, + scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: BTreeSet::from(["2026".to_string()]), + }, + producers: SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION.into_iter().collect(), + }]) + .expect("durable producer replay should encode"), + ) + .expect("durable producer replay should restore restart authority"); + let dirty_usage_snapshot = + snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation()); + let mut segment_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot) + .segment_invalidation_proof() + .expect("complete process-local producer coverage should produce proof metadata"); + segment_proof.cold_zero_walk_oracle = true; + let scan_plan_digest = DataUsageScanPlanDigest([6; 32]); + let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(0, 1)]); + let baseline = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + scanner_cycle: Some(7), + scanner_epoch: Some(11), + buckets_count: 2, + buckets_usage: HashMap::from([ + ("photos".to_string(), Default::default()), + ("archive".to_string(), Default::default()), + ]), + usage_snapshot_complete: true, + usage_snapshot_converged: Some(true), + usage_snapshot_set_states: expected_sources + .iter() + .map(|source| DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"), + set_index: u64::try_from(source.set_index).expect("test set index should fit"), + scanner_cycle: Some(7), + scanner_epoch: Some(11), + scan_plan_digest: Some(scan_plan_digest.0), + complete: true, + tombstone: false, + segment_invalidation_proof: Some(segment_proof.clone()), + }) + .collect(), + ..Default::default() + }; + let baseline = Bytes::from(serde_json::to_vec(&baseline).expect("baseline should encode")); + + let preflight = scanner_segment_reuse_activation_preflight_for_baseline( + &dirty_usage_snapshot, + false, + ScannerCacheBaselineProof { + authoritative_data: Some(&baseline), + observed_candidate_data: None, + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }, + ); + + assert!(preflight.production_activation); + assert!(preflight.scanner_segment_reuse_activated); + assert_eq!(preflight.fail_closed_blockers().collect::>(), Vec::<&str>::new()); + + let mut missing_cold_baseline = + serde_json::from_slice::(&baseline).expect("baseline should decode for negative case"); + missing_cold_baseline.usage_snapshot_set_states[0] + .segment_invalidation_proof + .as_mut() + .expect("proof should exist") + .cold_zero_walk_oracle = false; + let missing_cold_baseline = Bytes::from(serde_json::to_vec(&missing_cold_baseline).expect("negative baseline should encode")); + let preflight = scanner_segment_reuse_activation_preflight_for_baseline( + &dirty_usage_snapshot, + false, + ScannerCacheBaselineProof { + authoritative_data: Some(&missing_cold_baseline), + observed_candidate_data: None, + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }, + ); + + assert!(preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!( + preflight.fail_closed_blockers().collect::>(), + vec!["missing_cold_zero_walk_oracle"] + ); + + clear_dirty_usage_buckets(dirty_usage_snapshot.buckets.as_ref()); + record_dirty_usage_object_from_producer("photos", "2027/object", SegmentInvalidationProducerIdentity::PutObject); + let later_dirty_usage_snapshot = + snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation()); + let preflight = scanner_segment_reuse_activation_preflight_for_baseline( + &later_dirty_usage_snapshot, + false, + ScannerCacheBaselineProof { + authoritative_data: Some(&baseline), + observed_candidate_data: None, + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }, + ); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!( + preflight.fail_closed_blockers().collect::>(), + vec!["missing_durable_journal_replay"] + ); + + record_dirty_usage_bucket("photos"); + let unidentified_snapshot = + snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation()); + let preflight = scanner_segment_reuse_activation_preflight_for_baseline( + &unidentified_snapshot, + false, + ScannerCacheBaselineProof { + authoritative_data: Some(&baseline), + observed_candidate_data: None, + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }, + ); + assert!(!preflight.scanner_segment_reuse_activated); + assert!( + preflight + .fail_closed_blockers() + .any(|blocker| blocker == "missing_producer_identity") + ); + clear_dirty_usage_buckets_for_tests(); +} + +#[test] +fn scanner_cycle_result_returns_segment_reuse_activation_preflight() { + let proof = ScannerSegmentReuseActivationProof { + production_activation: true, + producer_identity_coverage_complete: true, + durable_producer_identity: true, + durable_dirty_producer_journal: true, + restart_gap_absent: true, + generation_window_bound: true, + overflow_absent: true, + ack_generation_guard: true, + cold_zero_walk_oracle: true, + distributed_peer_invalidation: true, + }; + let preflight = scanner_segment_reuse_activation_preflight_from_proof(proof); + + let result = ScannerCycleResult::new(ScannerCycleStatus::Complete, None).with_segment_reuse_activation_preflight(preflight); + + assert_eq!(result.segment_reuse_activation_preflight, preflight); +} + +fn assert_segment_reuse_activation_blocked_by(proof: ScannerSegmentReuseActivationProof, blocker: &'static str) { + let preflight = scanner_segment_reuse_activation_preflight_from_proof(proof); + + assert!(preflight.production_activation); + assert!(!preflight.scanner_segment_reuse_activated); + assert_eq!(preflight.fail_closed_blockers().collect::>(), vec![blocker]); +} + +fn complete_process_local_producer_evidence() -> DirtyUsageProducerEvidence { + DirtyUsageProducerEvidence { + producer_identity_coverage_complete: true, + durable_producer_identity: false, + durable_dirty_producer_journal: false, + restart_gap_absent: false, + generation_window_bound: true, + generation_start: 7, + generation_end: 7, + } +} + +fn complete_set_cache_with_segment_proof( + source: DataUsageCacheSource, + proof: crate::DataUsageSegmentInvalidationProof, +) -> DataUsageCache { + DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: 7, + last_update: Some(SystemTime::UNIX_EPOCH), + leader_epoch: 11, + source: Some(source), + snapshot_complete: true, + scan_plan_digest: Some(DataUsageScanPlanDigest([3; 32])), + segment_invalidation_proof: Some(proof), + ..Default::default() + }, + cache: HashMap::new(), + } +} + async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc) { init_ecstore_config_for_scanner_tests(); let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created"); @@ -1039,6 +1474,32 @@ fn dirty_usage_snapshot_detects_uncovered_generation() { clear_dirty_usage_buckets_for_tests(); } +#[test] +#[serial] +fn dirty_usage_producer_evidence_tracks_process_local_coverage_without_durable_restart_authority() { + use crate::segment_invalidation::SegmentInvalidationProducerIdentity; + + clear_dirty_usage_buckets_for_tests(); + record_dirty_usage_bucket_from_producers("photos", SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION); + let snapshot = snapshot_dirty_usage_buckets(&[bucket_info("photos")], dirty_usage_generation()); + + let evidence = dirty_usage_producer_evidence(&snapshot); + + assert!(evidence.generation_window_bound); + assert!(evidence.producer_identity_coverage_complete); + assert!(!evidence.durable_producer_identity); + assert!(!evidence.durable_dirty_producer_journal); + assert!(!evidence.restart_gap_absent); + assert_eq!(evidence.generation_start, snapshot.buckets["photos"]); + assert_eq!(evidence.generation_end, snapshot.buckets["photos"]); + + record_dirty_usage_bucket_from_producer("videos", SegmentInvalidationProducerIdentity::PutObject); + let stale_evidence = dirty_usage_producer_evidence(&snapshot); + assert!(!stale_evidence.generation_window_bound); + assert!(!stale_evidence.producer_identity_coverage_complete); + clear_dirty_usage_buckets_for_tests(); +} + #[test] fn generation_saturates_instead_of_wrapping() { let generation = AtomicU64::new(u64::MAX - 1); @@ -1346,6 +1807,13 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers let ctx = CancellationToken::new(); let empty_execution = DataUsageScanPlanDigest([5; 32]); + let segment_invalidation_proof = crate::DataUsageSegmentInvalidationProof { + process_epoch: scanner_activity_epoch().to_string(), + generation_start: 8, + generation_end: 8, + producer_identity_coverage_complete: true, + cold_zero_walk_oracle: false, + }; set.nsscanner_cache( ctx.clone(), ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()), @@ -1365,6 +1833,8 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers bucket_failures: ScannerBucketFailureState::default(), pending_maintenance_work: Arc::new(AtomicBool::new(false)), cache_cycle_floor: Arc::new(AtomicU64::new(8)), + cold_zero_walk_reuse_observed: Arc::new(AtomicBool::new(false)), + segment_invalidation_proof: Some(segment_invalidation_proof.clone()), }, tx, 8, @@ -1374,6 +1844,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers .expect("empty set scope should replace its prior nonempty cache"); let empty = rx.try_recv().expect("empty set snapshot should be published"); assert_eq!(empty.info.scan_execution_digest, Some(empty_execution)); + assert_eq!(empty.info.segment_invalidation_proof, Some(segment_invalidation_proof)); assert!(empty.info.snapshot_complete); let root = empty.checked_flatten(DATA_USAGE_ROOT).expect("complete empty root"); assert_eq!((root.size, root.objects), (0, 0)); @@ -1401,6 +1872,45 @@ fn complete_usage_baseline( scan_plan_digest: Some(scan_plan_digest.0), complete: true, tombstone: false, + segment_invalidation_proof: None, + }], + ..Default::default() + }; + bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode")) +} + +fn complete_segment_reuse_baseline( + source: DataUsageCacheSource, + scan_plan_digest: DataUsageScanPlanDigest, + scanner_cycle: u64, + scanner_epoch: u64, + evidence: DirtyUsageProducerEvidence, + buckets: &[&str], +) -> bytes::Bytes { + let mut proof = evidence + .segment_invalidation_proof() + .expect("durable producer evidence should produce segment proof"); + proof.cold_zero_walk_oracle = true; + let baseline = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(scanner_epoch), + buckets_count: u64::try_from(buckets.len()).expect("test bucket count should fit"), + buckets_usage: buckets + .iter() + .map(|bucket| ((*bucket).to_string(), Default::default())) + .collect(), + usage_snapshot_complete: true, + usage_snapshot_converged: Some(true), + usage_snapshot_set_states: vec![DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"), + set_index: u64::try_from(source.set_index).expect("test set index should fit"), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(scanner_epoch), + scan_plan_digest: Some(scan_plan_digest.0), + complete: true, + tombstone: false, + segment_invalidation_proof: Some(proof), }], ..Default::default() }; @@ -1732,6 +2242,12 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi "photos".to_string(), DirtyUsageBucketScope::TopLevelEntries(HashSet::from(["2026".to_string()])), )]); + let dirty_usage_snapshot = DirtyUsageSnapshot { + buckets: Arc::new(HashMap::from([("photos".to_string(), 7)])), + scopes: Arc::new(dirty_scopes.clone()), + generation: 7, + covers_all_pending: true, + }; let locally_scoped = scoped_scan_scope_from_dirty_buckets( ScannerBucketScanScope::default(), HashSet::from(["photos".to_string()]), @@ -1755,7 +2271,7 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi let distributed = resolve_remote_dirty_usage_scope( ScannerBucketScanScope::default(), - HashSet::from(["photos".to_string()]), + &dirty_usage_snapshot, remote_dirty_usage, &[bucket_info("photos")], ScannerCacheBaselineProof { @@ -1781,6 +2297,94 @@ fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exi "peer dirty state is not a distributed segment invalidation proof" ); assert_eq!(distributed.remote_dirty_usage_acknowledgements.len(), 1); + let evidence = distributed + .distributed_segment_invalidation_evidence + .expect("same-window peer snapshot and scoped ACK capability form distributed evidence"); + assert_eq!(evidence.peer_count, 1); + assert_eq!(evidence.dirty_peer_count, 1); + assert_eq!( + evidence.invalidation_domain, + crate::segment_invalidation::SegmentInvalidationDomain::DistributedEc + ); + assert!(evidence.distributed_ec_invalidation); + assert!(evidence.same_window_remote_proof); + assert!(evidence.all_peers_bound_to_generation_window); +} + +#[test] +#[serial] +fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_scope() { + clear_dirty_usage_buckets_for_tests(); + let source = DataUsageCacheSource::new(1, 2); + let expected_sources = HashSet::from([source]); + let scan_plan_digest = DataUsageScanPlanDigest([9; 32]); + let entries = BTreeSet::from(["2026".to_string()]); + let bytes = encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry { + bucket: "photos".to_string(), + generation: 7, + scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries }, + producers: crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION + .iter() + .copied() + .collect(), + }]) + .expect("durable dirty usage replay record should encode"); + replay_durable_dirty_usage_producer_record(&bytes).expect("durable dirty usage replay should restore producer proof"); + let dirty_usage_snapshot = + snapshot_dirty_usage_buckets(&[bucket_info("photos"), bucket_info("archive")], dirty_usage_generation()); + let evidence = dirty_usage_producer_evidence(&dirty_usage_snapshot); + let baseline = complete_segment_reuse_baseline(source, scan_plan_digest, 7, 11, evidence, &["photos", "archive"]); + let expected_peers = HashMap::from([( + "node-a:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-a".to_string(), + generation: 3, + pending: true, + }, + )]); + let remote_dirty_usage = verified_remote_dirty_usage( + &expected_peers, + vec![( + "node-a:9000".to_string(), + peer_dirty_usage_snapshot("instance-a", 3, true, &[("archive", 3)]), + )], + ) + .expect("fixture remote dirty usage should verify at bucket granularity"); + + let result = resolve_remote_dirty_usage_scope( + ScannerBucketScanScope::default(), + &dirty_usage_snapshot, + remote_dirty_usage, + &[bucket_info("photos"), bucket_info("archive")], + ScannerCacheBaselineProof { + authoritative_data: Some(&baseline), + observed_candidate_data: None, + expected_sources: &expected_sources, + leader_epoch: 11, + want_cycle: 8, + scan_plan_digest, + }, + ); + + assert_eq!( + result + .scope + .selected_buckets + .as_deref() + .expect("distributed reuse still selects both dirty buckets"), + &HashSet::from(["photos".to_string(), "archive".to_string()]) + ); + assert!( + result.scope.prefix_scope_for("photos").is_some(), + "durable local producer proof may activate local segment reuse after distributed ACK capability evidence" + ); + assert!( + result.scope.prefix_scope_for("archive").is_none(), + "remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix" + ); + assert_eq!(result.remote_dirty_usage_acknowledgements.len(), 1); + assert!(result.distributed_segment_invalidation_evidence.is_some()); + clear_dirty_usage_buckets_for_tests(); } fn peer_dirty_usage_snapshot( @@ -1827,7 +2431,7 @@ fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() ScannerPeerDirtyUsageExpectation { instance_id: "instance-b".to_string(), generation: 3, - pending: false, + pending: true, }, ), ]); @@ -1874,10 +2478,36 @@ fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() }, }, ], + peer_count: 2, + dirty_peer_count: 2, }) ); } +#[test] +fn verified_remote_dirty_usage_rejects_peer_snapshot_that_contradicts_activity_pending_state() { + let expected_peers = HashMap::from([( + "node-a:9000".to_string(), + ScannerPeerDirtyUsageExpectation { + instance_id: "instance-a".to_string(), + generation: 7, + pending: false, + }, + )]); + + assert!( + verified_remote_dirty_usage( + &expected_peers, + vec![( + "node-a:9000".to_string(), + peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]), + )], + ) + .is_none(), + "a clean activity window cannot authorize a dirty peer snapshot or scoped ACK" + ); +} + #[test] fn scanner_scoped_dirty_usage_ack_cost_threshold_is_single_protocol_batch() { let acknowledgement = |entry_count: usize| crate::scanner::ScannerDirtyUsageAcknowledgement { @@ -1930,7 +2560,12 @@ fn remote_dirty_usage_scope_resolution_falls_back_when_ack_batch_exceeds_thresho let result = resolve_remote_dirty_usage_scope( ScannerBucketScanScope::default(), - HashSet::new(), + &DirtyUsageSnapshot { + buckets: Arc::new(HashMap::new()), + scopes: Arc::new(HashMap::new()), + generation: 7, + covers_all_pending: true, + }, remote_dirty_usage, &all_buckets, ScannerCacheBaselineProof { @@ -1951,6 +2586,7 @@ fn remote_dirty_usage_scope_resolution_falls_back_when_ack_batch_exceeds_thresho result.remote_dirty_usage_acknowledgements.is_empty(), "full-scan fallback must not send a scoped ACK that peers would reject or split" ); + assert!(result.distributed_segment_invalidation_evidence.is_none()); } #[test] @@ -2092,6 +2728,11 @@ async fn distributed_scoped_scan_falls_back_when_remote_scoped_ack_capability_is assert_eq!(result.scope.selected_buckets.as_deref(), expected_buckets.as_ref()); assert_eq!(result.remote_dirty_usage_acknowledgements.len(), expected_ack_count); + assert_eq!( + result.distributed_segment_invalidation_evidence.is_some(), + capability, + "distributed evidence requires an authenticated scoped ACK capability probe" + ); } } @@ -2211,6 +2852,58 @@ fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() { assert_eq!((stable.size, stable.objects), (15, 2)); assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0))); assert_eq!(prepared.cache.info.scan_bucket_incarnations, current_incarnations); + let proof = prepared + .cold_bucket_reuse_proof + .as_ref() + .expect("cold bucket reuse should be explicitly bound"); + assert_eq!(proof.baseline_scan_plan_digest, baseline_digest); + assert_eq!(proof.source, DataUsageCacheSource::new(1, 2)); + assert_eq!(proof.bucket_incarnations, HashMap::from([("stable".to_string(), Uuid::from_u128(1))])); +} + +#[test] +fn scoped_set_scan_reuses_all_cold_buckets_with_matching_incarnations() { + let baseline_digest = DataUsageScanPlanDigest([1; 32]); + let current_digest = DataUsageScanPlanDigest([2; 32]); + let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("archive", 20)], baseline_digest); + old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "archive"]); + let current_incarnations = old_cache.info.scan_bucket_incarnations.clone(); + let all_buckets = vec![ + bucket_info_with_created_time("stable"), + bucket_info_with_created_time("archive"), + ]; + + let prepared = prepare_scoped_set_scan( + &old_cache, + &all_buckets, + &all_buckets, + &ScannerBucketScanScope { + selected_buckets: Some(Arc::new(HashSet::from(["dirty-on-another-set".to_string()]))), + selected_bucket_prefixes: None, + baseline_scan_plan_digest: Some(baseline_digest), + }, + ScannerSetCacheGeneration { + want_cycle: 8, + leader_epoch: 11, + tier_registry_generation: 13, + source: DataUsageCacheSource::new(1, 2), + scan_plan_digest: current_digest, + }, + Some(¤t_incarnations), + ) + .expect("a set with only cold buckets should reuse the complete bound baseline"); + + assert!(prepared.buckets.is_empty()); + assert_eq!(prepared.cache.find("stable").map(|entry| entry.size), Some(10)); + assert_eq!(prepared.cache.find("archive").map(|entry| entry.size), Some(20)); + assert_eq!( + prepared + .cold_bucket_reuse_proof + .as_ref() + .expect("all cold bucket reuse should carry incarnation proof") + .bucket_incarnations, + current_incarnations + ); } #[test] @@ -2283,6 +2976,31 @@ fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() { ) .is_none() ); + + let mut missing_entry = complete_set_usage_cache(&[("stable", 10)], baseline_digest); + missing_entry.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "new"]); + assert!( + prepare_scoped_set_scan( + &missing_entry, + &all_buckets, + &all_buckets, + &ScannerBucketScanScope { + selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))), + selected_bucket_prefixes: None, + baseline_scan_plan_digest: Some(baseline_digest), + }, + ScannerSetCacheGeneration { + want_cycle: 8, + leader_epoch: 11, + tier_registry_generation: 13, + source: DataUsageCacheSource::new(1, 2), + scan_plan_digest: DataUsageScanPlanDigest([4; 32]), + }, + Some(&missing_entry.info.scan_bucket_incarnations), + ) + .is_none(), + "an incarnation without a durable bucket entry must not authorize a cold skip" + ); } #[test] diff --git a/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs b/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs index c6bc58c3e..d974bd923 100644 --- a/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs +++ b/crates/scanner/src/scanner_io/tests/scoped_entry_fallback.rs @@ -84,7 +84,14 @@ async fn persist_baseline(store: &Arc, baseline: &DataUsageInfo) { // Every invocation uses the production default scope. Once durable bucket // incarnations are present, the expected walker set follows the resolved scope. -async fn run_entry(store: &Arc, cycle: u64, selected: Option<&str>, expect_walks: bool) -> DataUsageInfo { +async fn run_entry( + store: &Arc, + cycle: u64, + selected: Option<&str>, + expect_walks: bool, + expect_activation: bool, + expect_prefix_scope: bool, +) -> DataUsageInfo { let drives = drive_identities(store).await; let inventory = store .list_bucket_for_scanner(&BucketOptions::default()) @@ -138,11 +145,19 @@ async fn run_entry(store: &Arc, cycle: u64, selected: Option<&str>, exp .expect("entry cycle should finish within the fixture deadline") .expect("entry cycle should succeed"); assert_eq!(result.status, ScannerCycleStatus::Complete); + let activation_preflight = result.segment_reuse_activation_preflight; let scope = observed.await.expect("production resolver should report its decision"); assert_eq!( scope.selected_buckets.as_deref(), selected.map(|name| HashSet::from([name.to_string()])).as_ref() ); + if let Some(selected) = selected { + assert_eq!( + scope.prefix_scope_for(selected).is_some(), + expect_prefix_scope, + "resolved prefix scope must match activation replay for cycle {cycle}" + ); + } let usage = receiver.recv().await.expect("one candidate should be delivered"); assert!(receiver.recv().await.is_none(), "there must be exactly one terminal candidate"); assert!(usage.usage_snapshot_complete); @@ -174,6 +189,22 @@ async fn run_entry(store: &Arc, cycle: u64, selected: Option<&str>, exp actual, expected_walks, "each listed source/bucket must have exactly the expected real walks" ); + assert!(activation_preflight.production_activation); + assert_eq!(activation_preflight.scanner_segment_reuse_activated, expect_activation); + let activation_blockers = activation_preflight.fail_closed_blockers().collect::>(); + if expect_activation { + assert_eq!(activation_blockers, Vec::<&str>::new()); + } else if selected.is_some() && expect_walks { + assert!( + !activation_blockers.contains(&"missing_cold_zero_walk_oracle"), + "a complete scoped reuse cycle must carry the cold zero-walk oracle: cycle={cycle} selected={selected:?} blockers={activation_blockers:?}" + ); + } else { + assert!( + activation_blockers.contains(&"missing_cold_zero_walk_oracle"), + "unscoped or same-cycle cache reuse must not claim the cold zero-walk oracle: cycle={cycle} selected={selected:?} expect_walks={expect_walks} blockers={activation_blockers:?}" + ); + } assert_eq!( read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) .await @@ -189,32 +220,141 @@ async fn run_entry(store: &Arc, cycle: u64, selected: Option<&str>, exp usage } -#[tokio::test] +fn record_segment_dirty_usage(bucket: &str) { + for producer in crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION { + record_dirty_usage_object_from_producer(bucket, "hot-segment/object", producer); + } +} + +fn replay_segment_dirty_usage(bucket: &str) { + replay_dirty_usage( + bucket, + ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: BTreeSet::from(["hot-segment".to_string()]), + }, + ); +} + +fn replay_whole_bucket_dirty_usage(bucket: &str) { + replay_dirty_usage(bucket, ScannerDurableDirtyUsageReplayScope::WholeBucket); +} + +fn replay_dirty_usage(bucket: &str, scope: ScannerDurableDirtyUsageReplayScope) { + replay_durable_dirty_usage_producer_record( + &encode_durable_dirty_usage_producer_replay_record(vec![ScannerDurableDirtyUsageReplayEntry { + bucket: bucket.to_string(), + generation: dirty_usage_generation(), + scope, + producers: crate::segment_invalidation::SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION + .into_iter() + .collect(), + }]) + .expect("durable segment replay should encode"), + ) + .expect("durable segment replay should restore producer authority"); +} + +// The scoped fallback fixture keeps two EC pools and several scan futures live +// at once. Run the async cases on a dedicated stack so Linux libtest defaults +// exercise the assertions instead of aborting before the oracle finishes. +fn run_scoped_entry_fallback_test(thread_name: &'static str, test_fn: F) +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + 'static, +{ + let handle = std::thread::Builder::new() + .name(thread_name.to_string()) + .stack_size(32 * 1024 * 1024) + .spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("scoped entry fallback runtime should build"); + runtime.block_on(test_fn()); + }) + .expect("scoped entry fallback test thread should spawn"); + if let Err(payload) = handle.join() { + std::panic::resume_unwind(payload); + } +} + +#[test] #[serial] -async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks() { +fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks() { + run_scoped_entry_fallback_test( + "scanner-scoped-entry-planned-scope", + scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_case, + ); +} + +async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks_case() { let (_dir, store) = setup_two_pool_scanner_store().await; clear_dirty_usage_buckets_for_tests(); let hot = format!("hot-{}", Uuid::new_v4().simple()); let cold = format!("cold-{}", Uuid::new_v4().simple()); create_bucket(&store, &hot).await; create_bucket(&store, &cold).await; - record_dirty_usage_bucket(&hot); - let baseline = run_entry(&store, 1, None, true).await; + record_segment_dirty_usage(&hot); + replay_segment_dirty_usage(&hot); + let baseline = run_entry(&store, 1, None, true, false, false).await; persist_baseline(&store, &baseline).await; // Same-cycle Current remains a retry. The later cycle may skip the cold // bucket only after the prior complete set cache has durable incarnations. - run_entry(&store, 1, Some(&hot), false).await; - let usage = run_entry(&store, 2, Some(&hot), true).await; - assert_eq!(usage.buckets_usage[&hot].objects_count, 1); + run_entry(&store, 1, Some(&hot), false, false, false).await; + let usage = run_entry(&store, 2, Some(&hot), true, true, false).await; + persist_baseline(&store, &usage).await; + acknowledge_dirty_usage_generation(scanner_activity_epoch(), dirty_usage_generation()) + .expect("durable whole-cycle publication should acknowledge the initial producer window"); + put_and_settle(&store, &hot, "hot-segment/object").await; + record_dirty_usage_object_from_producer( + &hot, + "hot-segment/object", + crate::segment_invalidation::SegmentInvalidationProducerIdentity::PutObject, + ); + replay_segment_dirty_usage(&hot); + let usage = run_entry(&store, 3, Some(&hot), true, true, true).await; + assert_eq!(usage.buckets_usage[&hot].objects_count, 2); assert_eq!(usage.buckets_usage[&cold].objects_count, 1); - assert_eq!(usage.objects_total_count, 2); + assert_eq!(usage.objects_total_count, 3); + + persist_baseline(&store, &usage).await; + acknowledge_dirty_usage_generation(scanner_activity_epoch(), dirty_usage_generation()) + .expect("durable prefix publication should acknowledge the typed suffix"); + for index in 0..=MAX_DIRTY_USAGE_TOP_LEVEL_ENTRIES_PER_BUCKET { + record_dirty_usage_object_from_producer( + &hot, + &format!("overflow-{index}/object"), + crate::segment_invalidation::SegmentInvalidationProducerIdentity::PutObject, + ); + } + replay_whole_bucket_dirty_usage(&hot); + let usage = run_entry(&store, 4, Some(&hot), true, true, false).await; + assert_eq!(usage.objects_total_count, 3); + + persist_baseline(&store, &usage).await; + acknowledge_dirty_usage_generation(scanner_activity_epoch(), dirty_usage_generation()) + .expect("durable whole-bucket fallback should acknowledge the overflow window"); + record_dirty_usage_object_from_producer( + &hot, + "hot-segment/object", + crate::segment_invalidation::SegmentInvalidationProducerIdentity::Unknown, + ); + let usage = run_entry(&store, 5, Some(&hot), true, false, false).await; + assert_eq!(usage.objects_total_count, 3); clear_dirty_usage_buckets_for_tests(); } -#[tokio::test] +#[test] #[serial] -async fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker() { +fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker() { + run_scoped_entry_fallback_test( + "scanner-scoped-entry-invalid-baseline", + scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker_case, + ); +} + +async fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker_case() { let (_dir, store) = setup_two_pool_scanner_store().await; clear_dirty_usage_buckets_for_tests(); let hot = format!("hot-{}", Uuid::new_v4().simple()); @@ -223,7 +363,7 @@ async fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker( create_bucket(&store, &cold).await; record_dirty_usage_bucket(&hot); // The first real scan is also the missing persisted-baseline case. - let baseline = run_entry(&store, 1, None, true).await; + let baseline = run_entry(&store, 1, None, true, false, false).await; for (index, kind) in [ "malformed", "unconverged", @@ -256,28 +396,43 @@ async fn scoped_entry_fallback_rejects_invalid_persisted_baseline_at_the_walker( crate::save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes) .await .expect("negative baseline should persist"); - let usage = run_entry(&store, u64::try_from(index).expect("fixture cycle index should fit") + 2, None, true).await; + let usage = run_entry( + &store, + u64::try_from(index).expect("fixture cycle index should fit") + 2, + None, + true, + false, + false, + ) + .await; assert_eq!(usage.objects_total_count, 2, "{kind}"); assert_eq!(usage.buckets_usage[&cold].objects_count, 1, "{kind}"); } clear_dirty_usage_buckets_for_tests(); } -#[tokio::test] +#[test] #[serial] -async fn scoped_entry_fallback_covers_overflow_and_new_bucket_inventory() { +fn scoped_entry_fallback_covers_overflow_and_new_bucket_inventory() { + run_scoped_entry_fallback_test( + "scanner-scoped-entry-overflow-inventory", + scoped_entry_fallback_covers_overflow_and_new_bucket_inventory_case, + ); +} + +async fn scoped_entry_fallback_covers_overflow_and_new_bucket_inventory_case() { let (_dir, store) = setup_two_pool_scanner_store().await; clear_dirty_usage_buckets_for_tests(); let hot = format!("hot-{}", Uuid::new_v4().simple()); create_bucket(&store, &hot).await; record_dirty_usage_bucket(&hot); - let baseline = run_entry(&store, 1, None, true).await; + let baseline = run_entry(&store, 1, None, true, false, false).await; persist_baseline(&store, &baseline).await; for index in 0..=crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES { record_dirty_usage_bucket(&format!("overflow-{index}")); } assert!(dirty_usage_buckets_for_tests().len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES); - let usage = run_entry(&store, 2, None, true).await; + let usage = run_entry(&store, 2, None, true, false, false).await; assert_eq!(usage.objects_total_count, 1); clear_dirty_usage_buckets_for_tests(); @@ -285,7 +440,7 @@ async fn scoped_entry_fallback_covers_overflow_and_new_bucket_inventory() { let new_bucket = format!("new-{}", Uuid::new_v4().simple()); create_bucket(&store, &new_bucket).await; // Even a previously valid baseline cannot cover the changed inventory. - let usage = run_entry(&store, 3, None, true).await; + let usage = run_entry(&store, 3, None, true, false, false).await; assert_eq!(usage.objects_total_count, 2); assert_eq!(usage.buckets_usage[&new_bucket].objects_count, 1); clear_dirty_usage_buckets_for_tests(); diff --git a/crates/scanner/src/scanner_io/tests/service_cohort.rs b/crates/scanner/src/scanner_io/tests/service_cohort.rs index c720fb8e1..00ea5324c 100644 --- a/crates/scanner/src/scanner_io/tests/service_cohort.rs +++ b/crates/scanner/src/scanner_io/tests/service_cohort.rs @@ -16,23 +16,29 @@ use super::*; use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision}; async fn create_cohort_bucket(store: &ECStore, bucket: &str) { + create_cohort_bucket_objects(store, bucket, 1).await; +} + +async fn create_cohort_bucket_objects(store: &ECStore, bucket: &str, objects: usize) { store .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("fixture bucket"); for set in store.all_set_disks() { - let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec()); - set.put_object( - bucket, - "initial", - &mut reader, - &ScannerObjectOptions { - no_lock: true, - ..Default::default() - }, - ) - .await - .expect("fixture object and all rename tails should persist"); + for index in 0..objects { + let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec()); + set.put_object( + bucket, + &format!("object-{index:04}"), + &mut reader, + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("fixture object and all rename tails should persist"); + } } } @@ -168,6 +174,92 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources() { clear_dirty_usage_buckets_for_tests(); } +#[tokio::test] +#[serial] +async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() { + let (_dir, store) = setup_two_pool_scanner_store().await; + clear_dirty_usage_buckets_for_tests(); + let flat = format!("a-flat-{}", Uuid::new_v4().simple()); + let small = format!("z-small-{}", Uuid::new_v4().simple()); + create_cohort_bucket_objects(&store, &flat, 6).await; + create_cohort_bucket(&store, &small).await; + let cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default())); + + let expected_flat = store + .all_set_disks() + .iter() + .map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), flat.clone())) + .collect::>(); + let expected_small = store + .all_set_disks() + .iter() + .map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), small.clone())) + .collect::>(); + + let ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking( + &ctx, + ScannerCycleBudgetConfig { + max_objects: Some(1), + ..Default::default() + }, + ); + let (result, usage) = run_cohort_cycle(&store, cohort.clone(), 1, budget.clone()).await; + assert_eq!(result.status, ScannerCycleStatus::Incomplete); + assert!(usage.is_none(), "wide-bucket budget exhaustion must not publish a partial aggregate"); + assert!(budget.budget_elapsed()); + let first_round_admitted = cohort + .lock() + .expect("cohort lock") + .admitted_members() + .into_iter() + .collect::>(); + assert!( + !first_round_admitted.is_disjoint(&expected_flat), + "the first fixed budget round should exercise the wide flat bucket" + ); + assert!( + first_round_admitted.is_disjoint(&expected_small), + "a small bucket not yet reached by the real scanner must not be marked admitted" + ); + + for cycle in 2..=4 { + let ctx = CancellationToken::new(); + let (result, usage) = run_cohort_cycle( + &store, + cohort.clone(), + cycle, + ScannerCycleBudget::new_with_progress_tracking( + &ctx, + ScannerCycleBudgetConfig { + max_objects: Some(1), + ..Default::default() + }, + ), + ) + .await; + assert_eq!(result.status, ScannerCycleStatus::Incomplete); + assert!(usage.is_none(), "mixed partial coverage still cannot publish the set root"); + } + let admitted_after_budgeted_rounds = cohort + .lock() + .expect("cohort lock") + .admitted_members() + .into_iter() + .collect::>(); + assert!( + expected_small.is_subset(&admitted_after_budgeted_rounds), + "tracked small buckets must receive real execution opportunities within their fixed service-round bound" + ); + + let ctx = CancellationToken::new(); + let (result, usage) = + run_cohort_cycle(&store, cohort, 5, ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default())).await; + assert_eq!(result.status, ScannerCycleStatus::Complete); + assert_eq!(usage.expect("final complete aggregate").objects_total_count, 14); + clear_dirty_usage_buckets_for_tests(); +} + #[tokio::test] #[serial] async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() { diff --git a/crates/scanner/src/segment_invalidation.rs b/crates/scanner/src/segment_invalidation.rs index f2a74658c..308f3e6f6 100644 --- a/crates/scanner/src/segment_invalidation.rs +++ b/crates/scanner/src/segment_invalidation.rs @@ -25,6 +25,7 @@ pub enum SegmentInvalidationError { ByteLimit, InvalidProof, InvalidKey, + UnknownProducer, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -50,6 +51,102 @@ impl SegmentInvalidationProducer { ]; } +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SegmentInvalidationProducerIdentity { + PutObject, + DeleteObject, + DeleteMarker, + CompleteMultipartUpload, + AbortMultipartUpload, + ObjectMetadata, + BucketMetadata, + Replication, + TierTransition, + TierExpiration, + DirectoryObject, + Unknown, + TestFixture, +} + +impl SegmentInvalidationProducerIdentity { + pub const REQUIRED_PRODUCTION: [Self; 11] = [ + Self::PutObject, + Self::DeleteObject, + Self::DeleteMarker, + Self::CompleteMultipartUpload, + Self::AbortMultipartUpload, + Self::ObjectMetadata, + Self::BucketMetadata, + Self::Replication, + Self::TierTransition, + Self::TierExpiration, + Self::DirectoryObject, + ]; + + pub(crate) const REQUIRED_PRODUCTION_COVERAGE_MASK: u64 = (1_u64 << 11) - 1; + + pub(crate) const fn production_coverage_bit(self) -> Option { + match self { + Self::PutObject => Some(1_u64 << 0), + Self::DeleteObject => Some(1_u64 << 1), + Self::DeleteMarker => Some(1_u64 << 2), + Self::CompleteMultipartUpload => Some(1_u64 << 3), + Self::AbortMultipartUpload => Some(1_u64 << 4), + Self::ObjectMetadata => Some(1_u64 << 5), + Self::BucketMetadata => Some(1_u64 << 6), + Self::Replication => Some(1_u64 << 7), + Self::TierTransition => Some(1_u64 << 8), + Self::TierExpiration => Some(1_u64 << 9), + Self::DirectoryObject => Some(1_u64 << 10), + Self::Unknown | Self::TestFixture => None, + } + } + + pub fn producer(self) -> Option { + match self { + Self::PutObject => Some(SegmentInvalidationProducer::Put), + Self::DeleteObject => Some(SegmentInvalidationProducer::Delete), + Self::DeleteMarker => Some(SegmentInvalidationProducer::DeleteMarker), + Self::CompleteMultipartUpload | Self::AbortMultipartUpload => Some(SegmentInvalidationProducer::Multipart), + Self::ObjectMetadata => Some(SegmentInvalidationProducer::Put), + Self::BucketMetadata => Some(SegmentInvalidationProducer::DirectoryObject), + Self::Replication => Some(SegmentInvalidationProducer::Replication), + Self::TierTransition | Self::TierExpiration => Some(SegmentInvalidationProducer::Tier), + Self::DirectoryObject => Some(SegmentInvalidationProducer::DirectoryObject), + Self::Unknown | Self::TestFixture => None, + } + } +} + +pub fn complete_segment_invalidation_producers( + identities: I, +) -> Result, SegmentInvalidationError> +where + I: IntoIterator, +{ + let mut covered_identities = BTreeSet::new(); + let mut producers = BTreeSet::new(); + for identity in identities { + let Some(producer) = identity.producer() else { + return Err(SegmentInvalidationError::UnknownProducer); + }; + covered_identities.insert(identity); + producers.insert(producer); + } + if SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION + .iter() + .all(|identity| covered_identities.contains(identity)) + && SegmentInvalidationProducer::REQUIRED + .iter() + .all(|producer| producers.contains(producer)) + { + Ok(producers) + } else { + Err(SegmentInvalidationError::InvalidProof) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SegmentInvalidationDomain { LocalSingleSet, @@ -170,7 +267,8 @@ mod tests { use super::*; fn producers() -> BTreeSet { - SegmentInvalidationProducer::REQUIRED.into_iter().collect() + complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION) + .expect("production producer matrix should be complete") } fn envelope() -> SegmentInvalidationEnvelope { @@ -320,6 +418,82 @@ mod tests { ); } + #[test] + fn segment_invalidation_producer_identities_must_be_known_and_complete() { + let coverage_mask = SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION + .into_iter() + .filter_map(SegmentInvalidationProducerIdentity::production_coverage_bit) + .fold(0_u64, std::ops::BitOr::bitor); + assert_eq!(coverage_mask, SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION_COVERAGE_MASK); + assert_eq!(SegmentInvalidationProducerIdentity::Unknown.production_coverage_bit(), None); + assert_eq!(SegmentInvalidationProducerIdentity::TestFixture.production_coverage_bit(), None); + assert_eq!( + complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION), + Ok(SegmentInvalidationProducer::REQUIRED.into_iter().collect()) + ); + assert_eq!( + complete_segment_invalidation_producers([ + SegmentInvalidationProducerIdentity::PutObject, + SegmentInvalidationProducerIdentity::DeleteObject, + SegmentInvalidationProducerIdentity::DeleteMarker, + SegmentInvalidationProducerIdentity::CompleteMultipartUpload, + SegmentInvalidationProducerIdentity::AbortMultipartUpload, + SegmentInvalidationProducerIdentity::ObjectMetadata, + SegmentInvalidationProducerIdentity::BucketMetadata, + SegmentInvalidationProducerIdentity::Replication, + SegmentInvalidationProducerIdentity::TierTransition, + SegmentInvalidationProducerIdentity::TierExpiration, + SegmentInvalidationProducerIdentity::DirectoryObject, + SegmentInvalidationProducerIdentity::Unknown, + ]), + Err(SegmentInvalidationError::UnknownProducer) + ); + assert_eq!( + complete_segment_invalidation_producers([ + SegmentInvalidationProducerIdentity::PutObject, + SegmentInvalidationProducerIdentity::DeleteObject, + SegmentInvalidationProducerIdentity::DeleteMarker, + SegmentInvalidationProducerIdentity::CompleteMultipartUpload, + SegmentInvalidationProducerIdentity::AbortMultipartUpload, + SegmentInvalidationProducerIdentity::ObjectMetadata, + SegmentInvalidationProducerIdentity::BucketMetadata, + SegmentInvalidationProducerIdentity::Replication, + SegmentInvalidationProducerIdentity::TierTransition, + SegmentInvalidationProducerIdentity::TierExpiration, + SegmentInvalidationProducerIdentity::DirectoryObject, + SegmentInvalidationProducerIdentity::TestFixture, + ]), + Err(SegmentInvalidationError::UnknownProducer) + ); + assert_eq!( + complete_segment_invalidation_producers([ + SegmentInvalidationProducerIdentity::PutObject, + SegmentInvalidationProducerIdentity::DeleteObject, + SegmentInvalidationProducerIdentity::DeleteMarker, + SegmentInvalidationProducerIdentity::CompleteMultipartUpload, + SegmentInvalidationProducerIdentity::Replication, + SegmentInvalidationProducerIdentity::TierTransition, + SegmentInvalidationProducerIdentity::DirectoryObject, + ]), + Err(SegmentInvalidationError::InvalidProof) + ); + assert_eq!( + complete_segment_invalidation_producers([ + SegmentInvalidationProducerIdentity::PutObject, + SegmentInvalidationProducerIdentity::DeleteObject, + SegmentInvalidationProducerIdentity::DeleteMarker, + SegmentInvalidationProducerIdentity::CompleteMultipartUpload, + SegmentInvalidationProducerIdentity::AbortMultipartUpload, + SegmentInvalidationProducerIdentity::ObjectMetadata, + SegmentInvalidationProducerIdentity::BucketMetadata, + SegmentInvalidationProducerIdentity::Replication, + SegmentInvalidationProducerIdentity::TierTransition, + SegmentInvalidationProducerIdentity::DirectoryObject, + ]), + Err(SegmentInvalidationError::InvalidProof) + ); + } + #[test] fn segment_invalidation_entries_are_bounded_and_key_checked() { let envelope = envelope(); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 2faa2a33e..fbc5814b9 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -28,6 +28,13 @@ pub(crate) use s3s::dto::{ #[cfg(test)] pub(crate) use s3s::dto::{ExpirationStatus as EcstoreExpirationStatus, LifecycleRule as EcstoreLifecycleRule}; +pub(crate) use rustfs_ecstore::api::data_usage::{ + MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica, + register_scanner_pause_backlog_retirement_planner, +}; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::data_usage::{NativeScannerPauseBacklogWriteFault, SourceCleanupDeleteBarrier}; + pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys; pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc; pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{ @@ -135,6 +142,13 @@ use rustfs_storage_api as storage_contracts; pub(crate) type EcstoreHealResultItem = ::HealResultItem; pub(crate) mod owner { + pub(crate) use super::{ + MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica, + register_scanner_pause_backlog_retirement_planner, + }; + #[cfg(test)] + pub(crate) use super::{NativeScannerPauseBacklogWriteFault, SourceCleanupDeleteBarrier}; + #[cfg(test)] pub(crate) use rustfs_ecstore::api::set_disk::test_util::hold_namespace_commit as ecstore_hold_namespace_commit; diff --git a/docker-compose.yml b/docker-compose.yml index a28946575..bc76cdadf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: - RUSTFS_OBS_LOGGER_LEVEL=info - RUSTFS_TLS_PATH=/opt/tls - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev} volumes: - ./deploy/data/pro:/data - ./deploy/logs:/app/logs diff --git a/docs/README.md b/docs/README.md index 872b5b9be..35d3324af 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,3 +21,6 @@ operators should start with: Other runbooks remain grouped by filename in [`operations/`](operations/); architecture pages link to the relevant runbook where a cross-boundary procedure is required. + +For storage dashboards, see [Storage metrics and observer selection](operations/storage-metrics.md): +drive ownership, snapshot freshness, counter queries, and rolling upgrades. diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 0720cc8b3..e61e4e15b 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -38,9 +38,10 @@ - `put-file-auth-epoch-strict` internode put_file epoch compatibility: rc.2 peers can cache a remote put_file capability before that remote node restarts, then continue sending v1 authenticated uploads with the old server epoch; those peers cannot recover from the 409 conflict used by newer clients to trigger a re-probe. Servers temporarily accept signed, non-nil stale put_file epochs while legacy put_file auth remains non-strict so mixed-version rolling upgrades can finish multipart/object writes. Remove the stale-epoch fallback after the minimum supported RustFS peer version re-probes put_file capability after server-epoch conflicts and legacy put_file auth is no longer accepted. - `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC. - `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus. -- `backlog-1316` legacy encrypted multipart range seek: the feature remains opt-in until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently. +- `backlog-1316` legacy encrypted multipart range seek: the feature is on by default (RUSTFS_ENCRYPTED_RANGE_SEEK, default true) and the switch remains only as a kill switch until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently. - `tonic-013-status-render` peer RPC failure classification: internode failures that reach a node only as text (a peer's error_info payload, a status flattened through format!) are classified by matching the rendering of an Unavailable gRPC status. Releases up to 1.0.0-alpha.38 shipped tonic 0.13, which rendered that status as "status: Unavailable, message: ..."; tonic 0.14 renders it as "code: 'The service is currently unavailable', message: ...". Both forms are matched so an older peer's relayed text still marks an unreachable peer offline. Remove the tonic 0.13 form after the minimum supported RustFS peer version ships tonic 0.14 or later. - `rustfs-5063` pre-beta.9 Local KMS recovery: persisted Local KMS configs from beta.8 and earlier predate the explicit insecure-development flag, and encrypted key files use the legacy SHA-256 KDF. Remove the config fallback after supported upgrades have rewritten or explicitly resaved all pre-beta.9 configs with the development-default field, and remove the legacy KDF after supported upgrades have rewritten all pre-beta.9 Local KMS key files with explicit at-rest protection. +- `backlog-2369-legacy-nonce-fallback` pre-alpha.91 v1 segment nonce layout: releases before `1.0.0-alpha.91` reused a segment's part nonce for every block inside it, so the decrypt reader keeps that layout as its third and last v1 fallback. Because block zero's derived nonce equals that base nonce, the layout also lets a frame encrypted at index zero authenticate anywhere in its segment. Reads now lock a segment to whichever layout decoded its first non-zero block, which rejects a replay as soon as a later frame disagrees; a stream that is nothing but repeats of frame zero has no such later frame, so a deployment holding no pre-alpha.91 objects should set RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK=false to drop the layout outright. Remove the fallback (and the switch) after the minimum supported direct-upgrade release, and after migration tooling has rewritten every pre-alpha.91 encrypted object. - `sse-local-dek-json-v1` legacy local SSE DEK decoding: releases before the JSON envelope wrote wrapped DEKs as `base64(nonce):base64(ciphertext)`, so readers retain that decoder while all new writes use the versioned JSON envelope. Remove the colon decoder after the minimum supported direct-upgrade release writes JSON envelopes and migration tooling has rewritten every retained legacy object. - `not-initialized-error-code-v1` typed control-plane not-initialized wire code: control-plane RPC responses historically signaled an uninitialized peer only through the literal error_info string "errServerNotInitialized" (one drift site says "storage layer not initialized"). Responses now dual-carry a typed ControlPlaneErrorCode beside the legacy string, and clients prefer the code; the string stays populated and the client substring fallback (is_err_not_initialized, control_plane_failure) stays in place so mixed-version clusters keep classifying older peers' responses. Remove the substring fallback (and stop populating error_info for this case) after the minimum supported RustFS peer version always sends error_code. - `backlog-2097-tier-mutation-v4-error-text` tier-mutation Prepare rejection classification: a v3 server rejects a v4 request before store/runtime dispatch with the FailedPrecondition status and an authenticated, byte-exact unsupported-version message. A v4 coordinator recognizes only that exact code/message/requested-version tuple as definitely not persisted and fails the mutation without sending that peer an incompatible Abort; Unimplemented, near-text, missing/unknown failure classes, timeouts, and every other transport outcome remain ambiguous and stay in identity-bound Abort fanout. There is no automatic v3 retry. New servers retain v3 request/proof decoding for older coordinators, while operators must pause tier edit/remove/clear during a mixed v3/v4 rollout. Remove the text classifier after the minimum supported RustFS peer version returns the signed v4 PreDispatchRejected failure class. diff --git a/docs/architecture/heal-concurrency-model.md b/docs/architecture/heal-concurrency-model.md index f907b44fe..d7f2368d1 100644 --- a/docs/architecture/heal-concurrency-model.md +++ b/docs/architecture/heal-concurrency-model.md @@ -67,6 +67,12 @@ Heal-side invariants that hold regardless of the caller: - Read-repair's local TTL reservation dedups only its own source and does not block heals from other sources; the namespace lock is the backstop. - The healing flag is never persisted, so there is no reverse risk of a leftover marker making a later commit yield incorrectly. +## Graceful root-heal restart recovery + +Before a graceful shutdown cancels administrator cluster-wide heals, the manager saves unfinished requests on one coordinator disk as `.rustfs.sys/root-heal-.json`. Startup replays the same task IDs and remaining execution budgets. Completion, cancellation, and replacement by `force_start` retire the record conditionally. An uncertain write or deletion does not create a fallback copy; an unsuccessful handoff retains the unclean-shutdown marker. Invalid or unsupported records remain on disk and defer root recovery without blocking the existing replacement-recovery path. + +This handoff covers the same coordinator and storage topology while the record disk remains configured and readable. It does not migrate records when a pool is retired or provide failover after loss of that disk. Older versions do not understand these records; cancellation while downgraded cannot retire a newer version's pending record. If a terminal budget checkpoint cannot be written, the previous record is retained and a warning is logged; the remaining-budget guarantee requires that write to succeed. The format is separate from object metadata and erasure-set checkpoints. + ## Regression tests Both live in the test module of `crates/ecstore/src/set_disk/ops/heal.rs`: diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index 991dc6e8e..74d34275d 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -522,7 +522,7 @@ The following single-record protocol approves how historical objects without Rus ### Current -An absent `transitioned-version-state` key decodes as `TransitionVersionState::Unknown`. The current GET and free-version cleanup paths reject that state rather than interpreting an empty remote version as unversioned. There is no admin route that repairs this field in `xl.meta`. The existing transition-transaction reconcile route operates on expired `UploadOutcomeUnknown` transaction records and can exact-delete their canonical candidates; it is a separate protocol and must not be reused for metadata reconciliation. +An absent `transitioned-version-state` key decodes as `TransitionVersionState::Unknown`. Non-destructive compatibility reads distinguish legacy absence from explicit `Unknown`; empty-version reads require a bounded probe. Legacy free-version cleanup requires a persisted exact remote version and does not infer unversioned deletion from an empty field. The single-record Admin routes below inspect physical copies and live remote-version evidence. POST persists the proven three-field repair through exact-copy conditional metadata writes when every fleet member advertises policy version 5. An unsupported or unknown member produces `write_fence_unavailable`. The existing transition-transaction reconcile route operates on expired `UploadOutcomeUnknown` transaction records and can exact-delete their canonical candidates; it is a separate protocol and must not be reused for metadata reconciliation. An explicitly persisted `unknown`, a malformed state, conflicting RustFS/MinIO compatibility keys, an invalid or nil version identifier, and a partial transition tuple are not legacy absence. They remain invalid or ambiguous and fail closed. @@ -560,13 +560,13 @@ The POST may write only the derived `transitioned-version-state`, its correspond ### Outcome contract -POST returns exactly one of the following outcomes and whether it changed bytes. GET uses the same diagnostic names for non-applicable cases, returns `ready-to-migrate` when a missing state is provable, and returns `migrated` only when strong readback shows the record was already explicit and converged: +POST returns exactly one of the following outcomes. `changed` records confirmed writes; `changes_indeterminate` reports a failed write whose completion could not be established. After an indeterminate result, retry the same request or inspect again instead of assuming that no bytes changed. GET uses the same diagnostic names for non-applicable cases, returns `ready-to-migrate` when a missing state is provable, and returns `migrated` only when strong readback shows the record was already explicit and converged: | Outcome | Meaning and permitted effect | |---|---| | `migrated` | All authoritative copies already contain, or were monotonically advanced to, the same proven state and destination identity. Only this outcome makes the record eligible for later ordinary read/delete semantics. | | `retained-ambiguous` | The tuple is structurally legacy-compatible, but the live probe is missing, multiple, changing, unsupported, or otherwise cannot prove exactly one state. No metadata or remote object is changed. | -| `corrupt` | Explicit `Unknown`, malformed/contradictory compatibility keys, nil/invalid identifiers, partial transition metadata, or authoritative copies outside the one allowed `{original missing representation, exact proven target}` retry subset were observed. No backend probe is required after corruption is established, and nothing is changed. | +| `corrupt` | Explicit `Unknown`, malformed/contradictory compatibility keys, nil/invalid identifiers, partial transition metadata, or authoritative copies outside the one allowed `{original missing representation, exact proven target}` retry subset were observed. No backend probe is required after corruption is established. The writer stops further changes and retains any subset committed before a later conflict was discovered. | | `backend-unavailable` | The bound tier generation/destination cannot be acquired, the bounded probe fails, or a metadata quorum/strong readback needed to complete the operation is unavailable. Any already-persisted monotonic subset is retained for an idempotent retry; it is never rolled back. | HTTP failure detail may distinguish a stale expected tuple, lost fence, timeout, or unavailable quorum, but it must preserve one of these machine-readable outcomes. Logs and audit events include request identity, object identity, tier, generations, outcome, and whether bytes changed; they never include credentials or raw credential-derived configuration. @@ -588,11 +588,15 @@ The approved POST executes the following order. A step that cannot be proven sto Cross-pool and cross-set partial success is monotonic. The only legal repair edge is `missing state -> one proven {state, remote version, destination identity}`. A retry may accept an already-written subset only when every known copy equals the newly proven target, every remaining copy equals its original missing-state representation captured by the reconciliation digest, and all immutable source fields still match; it then fills only the missing copies. This exact target-plus-original subset is neither stale nor corrupt. The retry never clears a known state, rewrites it to another state, changes destination identity, or rolls a successful set back to missing/`Unknown`. Any other divergent value produces `corrupt`; an unavailable set/readback produces `backend-unavailable`, and destructive cleanup remains blocked until a later strong all-pool read proves complete convergence. +Each copy carries a raw SHA-256 `metadata_digest` and an `unchanged_metadata_digest` over canonical decoded metadata, all other versions, and all inline bytes, excluding only the three repairable suffixes on the selected version. An initial write requires the raw digest. An already-target retry requires the unchanged-content digest and the exact state/version/destination tuple. GET therefore reads the complete `xl.meta`, including inline payloads belonging to other versions; the reply contains digests, not payload bytes. + +Disk mutation serialization covers metadata updates, version writes, data publication, deletion, and rollback. Reconciliation refuses an outstanding rollback backup and bounds the object-directory backup check to 4,096 entries; it fails closed if that bound is exceeded. A stale rewrite of the same payload preserves the proven tuple or fails closed on a conflicting meaning. Each receiving disk acquires its own fleet/backend ownership, validates the conditional generation, and retains ownership through an atomic rename and mandatory directory fsync. A repair requires a Unix directory-sync implementation and metadata durability to be enabled for the bucket; unsupported platforms and disabled metadata sync block new repair writes instead of weakening the durability contract or overriding the configured policy. A zero-write validation pass enters the same disk mutation domain before final all-copy readback. Exact cleanup/recovery reads reject divergent transition bindings and cannot hide a minority owner behind majority absence. + GET takes the same fleet/topology snapshot and authoritative all-pool read but no write locks that imply mutation authority. Because GET is advisory, POST always repeats every fence, read, and live proof rather than promoting the GET result. ### Mixed-version and future batch work -The writer gate requires every node that can serve, rewrite, heal, decommission, or recover the affected `xl.meta` to preserve the explicit state and destination identity. A rolling fleet with an unknown/unsupported node is inspect-only. Downgrade is blocked while reconciled records could be rewritten by readers that erase or misinterpret those fields. Cross-pool movement must either copy the proven tuple unchanged or block reconciliation; a first-match lookup is never sufficient. +The writer gate requires every node that can serve, rewrite, heal, decommission, or recover the affected `xl.meta` to preserve the explicit state and destination identity. A rolling fleet with an unknown/unsupported node is inspect-only. A capability downgrade blocks further reconciliation admission. Deployment controls must also prevent running an older metadata writer against repaired records; the capability probe cannot prevent an operator from replacing the executable. Cross-pool movement must either copy the proven tuple unchanged or block reconciliation; a first-match lookup is never sufficient. Explicit `Unknown`, corruption, and ambiguity remain fail closed for reads that cannot prove non-destructive semantics and for every destructive path. A migrated record becomes ordinary explicit metadata, but reconciliation itself never transfers remote DELETE ownership. diff --git a/docs/architecture/readiness-matrix.md b/docs/architecture/readiness-matrix.md index 75b180bb6..a6fccc77c 100644 --- a/docs/architecture/readiness-matrix.md +++ b/docs/architecture/readiness-matrix.md @@ -39,7 +39,7 @@ an unknown or unsupported peer-health snapshot degrades readiness with - Liveness reports process availability and must not depend on storage, IAM, lock quorum, or peer health. -- Node readiness reports local dependency readiness. +- Node readiness reports the node's observed storage write quorum, local pool metadata write gate, IAM, and lock quorum. Runtime storage diagnostics do not change the existing startup `FullReady` publication gate or S3 request admission. - A blocked pool metadata writer degrades node and cluster-write readiness with `pool_meta_write_blocked`. Metadata save-gate inspection is bounded to 100 ms; contention reports `pool_metadata_check_timeout` without installing a block. @@ -49,11 +49,37 @@ an unknown or unsupported peer-health snapshot degrades readiness with decision. Runtime readiness and gate status are separate bounded observations. - Cluster write readiness requires write quorum and the runtime dependency readiness used by `FullReady`. -- Cluster read readiness may use the read-quorum path and cluster-health - timeout behavior. +- Cluster read readiness uses the storage read-quorum path and cluster-health timeout behavior. Its lock dependency still uses the per-set majority/write-lock health check. Actual shared namespace locks require `ceil(lock_clients / 2)`, so the cluster read probe is conservative: a four-client set can still admit some reads with two clients while the probe returns 503. This diagnostic change does not lower that probe's lock threshold. - `HEAD` health probes keep header/status semantics and do not require response bodies. +### Storage Detail Contract + +The existing `details.storage.ready` boolean and `connected` / `disconnected` status values are retained. `readinessScope` states the condition they summarize: + +| Probe | `readinessScope` | `source` | +| --- | --- | --- | +| `/health/ready`, `/minio/health/ready` | `write_quorum_and_pool_metadata` | `local_runtime` | +| `/minio/health/cluster` | `write_quorum_and_pool_metadata` | `storage_inventory` | +| `/minio/health/cluster/read` | `read_quorum` | `storage_inventory` | + +Node readiness additionally reports `details.storage.readQuorum`, `details.storage.writeQuorum`, and `details.poolMetadata.ready`. The metadata component's status is `writable` or `unavailable`; existing typed degradation reasons distinguish a write block from an inspection timeout. A healthy metadata writer alone no longer makes the storage component ready. + +Node storage quorum uses configured drives per set, all configured pools/sets, and their Standard storage-class data/parity layout. Missing, duplicate, unreachable, or unhealthy disk observations cannot supply extra quorum votes. The read quorum is the data-drive count; the write quorum is that count plus one when data and parity counts are equal. These are observations of available storage slots, not guarantees that a particular object's metadata, shards, or required locks are available. + +For a healthy IAM and metadata writer in a four-node, one-drive-per-node EC 2+2 set: + +| Surviving nodes | Storage read quorum | Storage write quorum | Pool metadata ready | Node HTTP / top-level ready | +| --- | --- | --- | --- | --- | +| 4 or 3 | true | true | true | 200 / true | +| 2 | true | false | true | 503 / false | +| 1 | false | false | true | 503 / false | +| All restored | true | true | true | 200 / true | + +The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection retains its own 100 ms budget. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy. + +The new fields are additive. Their absence in an older response is not evidence of storage quorum. Minimal responses still contain only the existing top-level fields, liveness remains dependency-independent, and HEAD responses remain bodyless. + ## Preservation Rules - Do not move peer-health checks into the S3 data hot path. diff --git a/docs/architecture/s3-compatibility-matrix.md b/docs/architecture/s3-compatibility-matrix.md index f296003dd..05b57c074 100644 --- a/docs/architecture/s3-compatibility-matrix.md +++ b/docs/architecture/s3-compatibility-matrix.md @@ -38,6 +38,12 @@ Counts ignore blank lines and comments; compute them from the files. The lifecyc "Supported" for the SSE row means RustFS encrypts and decrypts its own objects. MinIO SSE objects (SSE-S3, SSE-KMS, SSE-C) are not readable in default builds; see [minio-file-format-compat.md Part C](minio-file-format-compat.md#part-c--server-side-encryption-sse) for the `rio-v2` migration build. +### Client metadata expectations + +`CopyObject` with `MetadataDirective=REPLACE` clears standard metadata fields that the request omits, including `Content-Type`; it does not retain the source type or infer a default. Clients requiring a MIME type on the copied object must send `Content-Type` with the replacement metadata. This contract is covered by `crates/e2e_test/src/copy_object_metadata_test.rs`. A client test that expects an implicit `application/octet-stream` does not match this behavior. + +The MinIO-style `metadata=true` listing extension returns user metadata names without the HTTP `x-amz-meta-` prefix. It is not the standard S3 `ListObjectsV2` response. Clients that expect canonical HTTP header names in `UserMetadata` must normalize the names at that boundary; ordinary HEAD/GET metadata is unaffected. See `rustfs/src/app/bucket_usecase.rs` and its serialization tests. + ## Replication Support Boundary Site replication and bucket replication are not the same compatibility claim. diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index d0bf7ee7b..3ef65348d 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -15,6 +15,16 @@ RustFS ships several KMS backends. They differ not only in deployment effort but | Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | | AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments rooted in AWS IAM — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first | +## No KMS configured: the SSE-S3 local master key + +A deployment that never configures a KMS can still serve **SSE-S3** by setting `RUSTFS_SSE_S3_MASTER_KEY` to a base64-encoded 32-byte key. Data keys are then wrapped with that key using AES-256-GCM, on the node that serves the write. Understand three consequences before relying on it: + +- **The key is the whole confidentiality boundary.** It lives in the process environment of every node, with no ACL, no audit trail and no policy engine in front of it. +- **Objects written this way can never be rotated.** There is no key record to rotate and no rewrap path; changing the value makes every object written under the old one unreadable. Migrating to a KMS later means rewriting those objects (for example with CopyObject), not reconfiguring. +- **It does not serve SSE-KMS.** A request for `x-amz-server-side-encryption: aws:kms` on a node with no running KMS is refused — `400 InvalidRequest` when KMS was never configured, `503` when a configured service is not running. Earlier releases silently wrapped the data key with the local master key while still stamping `aws:kms` and the requested key id into the object metadata; that metadata claimed a KMS protection the object never had. If a deployment depended on that, either configure a KMS or ask for `AES256`. + +The value is unset by default, and a deployment that neither configures a KMS nor sets it simply cannot serve SSE-S3 (the write is refused, never silently downgraded to plaintext). + ## Migrating from MinIO: encrypted objects do not carry over > **Warning: default RustFS builds fail closed on objects that MinIO encrypted.** This applies to SSE-S3, SSE-KMS, and SSE-C, whichever KMS backend you configure; configuring `Static` with MinIO's key material does not make them readable. Such objects list and HEAD normally (their `xl.meta` parses), and only the payload read fails — with S3 `InvalidObjectState`, never plaintext. Read a sample of encrypted objects, not just their listings, before decommissioning the MinIO deployment. @@ -117,6 +127,37 @@ Decryption loads exactly the version recorded in the envelope and fails closed w Do not rotate any key until **every** RustFS node runs a build that understands the `master_key_version` envelope field. Older binaries ignore the field and always decrypt with the current material: harmless while nothing has been rotated, but after a rotation they fail to decrypt every object wrapped by an earlier key version. Complete the rolling upgrade of the entire cluster first, then rotate. The rest of this constraint class is collected in [Mixed-version clusters during a rolling upgrade](#mixed-version-clusters-during-a-rolling-upgrade). +## SSE-C requires a secure transport + +An SSE-C request carries the customer's AES key in a request header, so AWS S3 and MinIO both refuse one that did not arrive over TLS. A plaintext hop hands that key to anyone on the path, and because the object cannot be read without the same key, the exposure lasts as long as the object does. + +This release reports rather than refuses, because flipping straight to a rejection would break every plaintext staging and test deployment inside a release window: + +- Every SSE-C request on a plaintext transport increments `rustfs_ssec_plaintext_requests_total` and logs one `ssec_request_without_tls` warning per process. +- `RUSTFS_SSE_C_REQUIRE_TLS=true` (default `false`) refuses those requests now, with the same `400 InvalidRequest` wording AWS uses. Confirm the counter reads zero before enabling it. +- The default is expected to flip in a later release. + +The verdict is per connection: a listener that terminates TLS satisfies it, and so does an `https` protocol forwarded by a proxy the trusted-proxy configuration accepts. A direct plaintext client asserts nothing, and a forwarded protocol from an untrusted peer is not consulted. + +## Object ciphertext format: what the v1 frame layout does and does not authenticate + +Every object RustFS writes today uses the **v1** frame layout (the v2 layout exists and is read automatically, but its write switch `RUSTFS_ENCRYPTION_FRAME_V2` is off by default). Each frame is authenticated with AES-256-GCM under a nonce derived from the object's base nonce and the frame's index. Three properties do **not** follow from that, and an operator's threat model has to account for them: + +- **No frame-index binding.** A frame's index is not part of its associated data. Authentication proves a frame was produced under this object's key; it does not by itself prove the frame belongs at the position it occupies. +- **No final-frame authentication.** Nothing in a v1 stream marks the last frame, so a stream that has been cut short is not distinguishable from a shorter object by cryptographic means. +- **Truncation is not detected server-side.** A full GET is cut off by the length gate mid-stream and surfaces as `IncompleteBody` — after the response headers have already gone out. A ranged GET that ends early looks like an ordinary EOF and is not reported at all. A client that needs a truncation signal must compare the delivered length against `Content-Length` itself. + +These matter only to an attacker who can already rewrite the underlying shards. Shard integrity uses a keyed-hash-free checksum (HighwayHash), which such an attacker can recompute, so it is not a barrier. + +Two historical shapes additionally reuse a GCM nonce and cannot be repaired by any read-side change: + +| Shape | Written by | Consequence | Migration | +| --- | --- | --- | --- | +| Multipart objects written before `1.0.0-alpha.91` | The pre-alpha.91 writer reused a segment's part nonce for every block in it | The whole segment shares one nonce; a frame from index zero authenticates anywhere in that segment | Rewrite in place with CopyObject; then set `RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK=false` | +| SSE-C objects written before `1.0.0-beta.9` that carry no stored IV | The nonce was derived deterministically from bucket and key | Historical versions of the same key share a nonce, which affects confidentiality as well as forgeability | Rewrite in place with CopyObject | + +The decrypt reader locks each segment to whichever nonce layout decoded its first non-zero-index frame, so a replayed frame is rejected as soon as any later frame disagrees. A stream that is nothing but repeats of frame zero has no later frame to disagree, so a deployment that holds no pre-alpha.91 objects should set `RUSTFS_ENCRYPTION_LEGACY_NONCE_FALLBACK=false` (default `true`) to drop that layout entirely. Turning it off refuses to decrypt pre-alpha.91 objects, so migrate first. + ## Mixed-version clusters during a rolling upgrade During a rolling upgrade KMS state is shared three ways: **Vault** holds key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and can disagree on configuration for as long as the operator leaves them running, because the reload broadcast that converges configuration is one of the things an older build rejects. This section is written for the KV2 and Transit backends; the Local backend is unsupported for multi-node deployments regardless of version (see the [deployment support matrix](#deployment-support-matrix)). @@ -237,6 +278,7 @@ The Local backend stores one JSON record per key (`.key`) plus an Argon2 - `Local` is the default backend (`kms_backend` defaults to `local`) and is a development, testing and demo backend; it is not supported for production. Activating a backend whose capabilities report `production_supported: false` logs a `kms_backend_positioning` warning on every start, restart and reconfigure, and the `kms/status` capability matrix carries the same flag. The positioning is a warning, not a gate. - Configuration validation enforces stricter rules outside explicit development mode: a master key is required and `key_dir` must not live under the process temp directory. - The RustFS Kubernetes operator places the key directory on a PersistentVolumeClaim, so keys survive pod rescheduling. +- **A multi-node deployment cannot share it.** Key material lives on each node's own disk and the Argon2id salt is generated per node, so two nodes derive different keys from the same `master_key`. An object encrypted on node A cannot be decrypted on node B; behind a load balancer that appears as intermittent 500s on reads that succeeded a moment earlier. Configuring `Local` while the deployment is distributed logs `kms_node_local_backend_in_distributed_deployment` and appends the same warning to the `kms/configure` response. This stays a warning, not a gate. - Production multi-node deployments should use the Vault Transit backend. ### Deployment support matrix diff --git a/docs/operations/kms-disaster-recovery-drill.md b/docs/operations/kms-disaster-recovery-drill.md index 601ca0836..ae88413e8 100644 --- a/docs/operations/kms-disaster-recovery-drill.md +++ b/docs/operations/kms-disaster-recovery-drill.md @@ -9,12 +9,14 @@ The drill rehearses the complete loop — back up, lose the persistence layer, p The drill covers the **Local** backend, the only backend RustFS produces a full-material bundle for. The responsibility split is described in `crates/kms/src/backup/capability.rs`: -| Backend | What a RustFS bundle carries | What restores it | +| Backend | RustFS bundle export | What restores it | | --- | --- | --- | | Local | Key records, all stored versions, the KDF salt, sanitized configuration | The RustFS restore in this runbook | -| Static | Non-sensitive references only | The operator re-supplies the secret out of band | -| Vault KV2 + Transit | KV metadata and Transit ciphertext references | Vault's native snapshot restore, then the RustFS orchestration | -| Vault Transit | Metadata, configuration references, verification data | Vault's native snapshot restore, then the RustFS orchestration | +| Static | Refused with `501`; RustFS holds no material to export | The operator re-supplies the secret out of band | +| Vault KV2 | Refused with `501` | Vault's native snapshot restore, then the RustFS orchestration | +| Vault Transit | Refused with `501` | Vault's native snapshot restore, then the RustFS orchestration | + +The `501` is not a gap in this runbook: `POST /rustfs/admin/v3/kms/backup` refuses any backend other than `Local` (`rustfs/src/admin/handlers/kms_backup.rs`, `execute_backup`), so no RustFS bundle exists to plan around for the other three. Note that `capability.rs` still *declares* `FullMaterial` responsibility for Vault KV2 in storage-only mode; no export path implements it, so treat the declaration as a reservation, not a capability. For the Vault backends there is no RustFS-side export: the cryptographic root is non-exportable and comes back through Vault's own disaster-recovery flow. RustFS owns the refusal to proceed before that has happened and the ordering of everything after it — see the Vault section below. diff --git a/docs/operations/kms-observability-runbook.md b/docs/operations/kms-observability-runbook.md index 26a4cab7c..42d308fe6 100644 --- a/docs/operations/kms-observability-runbook.md +++ b/docs/operations/kms-observability-runbook.md @@ -225,6 +225,10 @@ KMS configured through the admin API is persisted to cluster storage and restore To recover from `load_failed` — or from any state where the server runs but its in-memory KMS lags the persisted configuration — call `POST /rustfs/admin/v3/kms/reload` (`kms:ServiceControl`). It re-reads the persisted configuration from cluster storage and reconfigures the service without resubmitting secrets, then broadcasts the reload to peer nodes. If reload keeps failing, check cluster storage health first (the read needs quorum), then `RUSTFS_KMS_CONFIG_SECRET`: an unseal error means the secret is missing or differs from the one that sealed the persisted copy — it must be identical on every node. +Reload short-circuits only when this node is **already running** the persisted configuration. A node whose KMS failed to start keeps that configuration and sits in `Error`, so reload reconfigures it — which starts the service — rather than reporting success while the node stays down. The same holds on every peer, which reaches the same path through the reload broadcast. + +**Cluster-wide versus node-local routes.** `configure`, `reconfigure` and `reload` are cluster operations: the node that serves the request broadcasts to its peers. `start` and `stop` are node-local and are **not** broadcast. Calling `POST /rustfs/admin/v3/kms/stop` through a load balancer therefore stops whichever node answered and leaves the cluster in a mixed state; address a specific node directly when you mean node-local semantics, and expect a later cluster-wide `reload` to start a stopped node again. + A separate event, `kms_config_load_skipped` with `reason="storage_uninitialized"`, comes from the ambient loader used by the peer-reload RPC path; seeing it outside a peer reload indicates a request arrived before storage initialization finished. ## Threshold calibration diff --git a/docs/operations/rolling-restart.md b/docs/operations/rolling-restart.md index ce185e2a2..6dbd4297a 100644 --- a/docs/operations/rolling-restart.md +++ b/docs/operations/rolling-restart.md @@ -56,8 +56,8 @@ When the whole cluster (or several nodes) went down and nodes are brought back o | `startup_finalization` | Last startup steps are being published. | 2. Logs say what the node waits for. The IAM recovery loop retries with backoff and logs `event="iam_bootstrap_retry_failed"` with an actionable `hint` field (for example, "storage read quorum not met yet; waiting for enough cluster nodes/disks to come online"). After repeated failures the level escalates from WARN to ERROR; this still does not kill the process. -3. Recovery is automatic. As soon as enough peers are online for the storage read quorum, the pending nodes finish IAM bootstrap on the next retry and flip `/health/ready` to `200` on their own. Restarting them does not speed anything up. -4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) return per-dependency detail during degradation; the `details` object shows `storage` / `iam` / `lock` readiness and `degradedReasons` lists machine-readable causes such as `storage_quorum_unavailable` or `lock_quorum_unavailable`: +3. Recovery is automatic. Once storage read quorum is available, pending nodes can finish IAM bootstrap on the next retry. `/health/ready` returns `200` when storage write quorum, the metadata write gate, IAM, and lock readiness are satisfied. Restarting pending nodes does not speed this up. +4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) separate `storage` / `poolMetadata` / `iam` / `lock` readiness. `storage.ready` summarizes write quorum plus the metadata write gate; `storage.readQuorum` and `storage.writeQuorum` show the separate quorum observations. A healthy `poolMetadata.ready` does not imply storage quorum. `degradedReasons` lists machine-readable causes such as `storage_quorum_unavailable`, `storage_and_lock_unavailable`, or `pool_metadata_check_timeout`. See the [storage detail contract](../architecture/readiness-matrix.md#storage-detail-contract) for probe scopes and sampling limits: ```bash curl -s http://:9000/health/ready | jq diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index f4ed5ffa8..9d245a641 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -39,8 +39,8 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/ The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`, `multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least three A1/B1/B2/A2 groups for both baseline/candidate with background work on, -and candidate-only background off/on. A measured leg lasts at least 900 -seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles). +and candidate-only background off/on. A measured release leg lasts at least 7200 +seconds; the minimum matrix contains 120 legs (240 hours before setup/oracles). The existing `performance-ab.yml` supplies the pattern for immutable build provenance and failure propagation, but its short Warp workload is not this scanner gate. No scheduled workflow starts this matrix automatically. @@ -63,9 +63,10 @@ The manifest has the following JSON contract (all fields are required): | Field | Value | |---|---| | `schema`, `evidence` | `1`, and `measured` or `synthetic`. | -| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. | +| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 7200..86400 seconds for measured release runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. | | `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. | | `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. | +| `release_evidence` | Required for `measured` runs. It binds the 3x4 EC8+4 topology, multi-pool/multi-set coverage, per-node metrics endpoints, same-window distributed sampling, process restart and crash-restart fault modes, mixed-version reader/writer/rollback participation, and allocation/flamegraph/RSS/save-frequency profile artifact requirements. Synthetic runs do not need this field and still cannot approve release evidence. | | `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. | | `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. | @@ -75,6 +76,14 @@ object/version/content result. Fix the foreground arrival rate (offered load), cache preparation procedure, configuration, and hardware across every leg. Do not include credentials in the manifest, adapter output, or saved commands; the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment. +The adapter must echo the measured run's `release_evidence` object in every +measurement response. A mismatch fails the cell because it means the deployment, +mixed-version set, crash mode, or profiler contract no longer matches the +operator-reviewed manifest. This echo is provenance binding only; it does not +replace the independent correctness oracle, distributed metrics samples, profile +artifacts, or ABBA comparison thresholds. The summary tool revalidates the same +manifest contract before it can print a measured PASS result, so hand-built or +trimmed reports without this provenance fail closed. #### Deployment Adapter Contract @@ -150,6 +159,14 @@ delay counts are both retained so an operator can reject unrelated or process-lifetime counter contamination. Correct repair oracles and the existing regression limits still apply in every case. +The `running-heal` build comparison also records a `w11` section for the bounded +retry-window evidence. `status=observed` requires same-window healthy-page +latency improvement, reduced heal lock-wait p99, bounded candidate RSS growth, +and a candidate attempt-cost value. `rss_regression` means latency and lock-wait +improved but RSS exceeded the allowed growth limit; `no_measured_benefit` means +attempt-cost evidence exists without the full W11 benefit; `pending` means the +attempt-cost evidence needed for the comparison is missing. + For P2, `measure.convergence` contains booleans `writes_stopped`, `last_mutation_observed`, `first_complete_publication`; numeric `last_mutation_time`, `last_mutation_observed_time`, `writes_stopped_time`, `window_start`, `window_end`, @@ -204,6 +221,17 @@ The command prints only `PASS scanner_heal_perf ...` for measured passing ABBA evidence, otherwise `FAIL scanner_heal_perf ...`. The JSON and Markdown outputs carry the key p99/throughput/P1/P2/cache-cost fields and artifact provenance hashes; raw per-cell logs remain in the original artifact tree for audit. +Failed or interrupted ABBA reports that contain only `status`, `performance`, +`completed_cells`, and `error` also summarize as `FAIL`; they do not become +performance evidence, and a missing comparison matrix is accepted only for a +non-passing report. +Measured passing reports must also retain the W10/W11 foreground-pressure, +heal-lock-wait, and heal-attempt-cost fields emitted by the ABBA evaluator. If +those fields are removed, empty, malformed, or length-mismatched, the quiet +summary fails closed instead of treating the report as performance evidence. For +`running-heal` build comparisons, the summary additionally requires the emitted +W11 section to be `observed` and to retain the RSS-growth, lock-wait, +healthy-page-latency, and candidate attempt-cost values. They cover the complete 120-cell schedule, data isolation, missing builds and oracles, zero samples/requests, swallowed request errors, offered-load drift, diff --git a/docs/operations/storage-metrics.md b/docs/operations/storage-metrics.md new file mode 100644 index 000000000..d3b22bea4 --- /dev/null +++ b/docs/operations/storage-metrics.md @@ -0,0 +1,142 @@ +# Storage metrics and observer selection + +**Use this when:** configuring storage dashboards, migrating from replicated +per-drive metrics, or diagnosing stale and missing disk observations. + +## Ownership and identity + +The storage collectors have two distinct scopes: + +| Scope | Metric families | Meaning | +|---|---|---| +| `collection_scope="local"` | `rustfs_system_drive_*`, `rustfs_node_disk_*` | Detailed metrics for drives owned by the reporting node; includes configured offline slots. Drive counts are local counts. | +| `collection_scope="cluster"` | Cluster capacity, health, objects/buckets, erasure sets, and `rustfs_cluster_drive_*` | One reporting node's observation of the whole cluster. | + +`observer` identifies the reporting node. `server` identifies the drive owner; +`drive` alone is not a unique disk key. Keep `rustfs_cluster_id`, `server`, and +`drive` when grouping drives, and retain pool/set/drive indices for topology. +Drive counters also carry `disk_id`, so a physical replacement starts a separate +series. Unknown IDs are empty; `rustfs_system_drive_info` is emitted only when +the ID and topology are known. `rustfs_system_drive_present` preserves a configured +slot even when its disk is disconnected and its ID is unavailable. + +Global drive metrics contain membership, runtime state, capacity and its source. +They do not duplicate remote API counters, error counters, or detailed I/O metrics. +An unreachable peer remains in the global inventory with the storage layer's +unknown/offline state and stale/missing capacity provenance. Missing capacity is +omitted from per-drive byte metrics. Cluster capacity may still include cached +observations: check `capacity_stale_drives` and `capacity_missing_drives` before +interpreting it. The existing cluster offline count includes drives not currently +observed online; use global per-drive runtime states to distinguish unknown. + +## Configure the pipeline + +Give every node of a deployment the same stable, unique resource attribute: + +```bash +OTEL_RESOURCE_ATTRIBUTES=rustfs.cluster.id=production-a +``` + +Keep existing resource attributes in the comma-separated value. The example +Compose files accept `OTEL_RESOURCE_ATTRIBUTES` and default to the development +cluster ID `rustfs-dev`; set a distinct ID for each deployment. The Collector's +`resource_to_telemetry_conversion.enabled: true` promotes it to +`rustfs_cluster_id` on scraped samples. Prometheus `external_labels` are not a +substitute: they do not add a cluster label to the local time series. + +Use the shipped Collector configuration with `send_timestamps: true` and keep +Prometheus's default `honor_timestamps: true`. Load +`.docker/observability/prometheus-rules/rustfs-storage.yml`. Use one ingestion +route per deployment/node; scraping replicas of the same Collector requires a +separate HA deduplication policy. Keep node and Prometheus clocks synchronized. + +Storage export callbacks read the current snapshot from memory and stop exporting +removed series. They do not probe storage or run peer RPCs. A successful collection +updates `rustfs_storage_snapshot_last_success_timestamp_seconds`; OTLP export +alone does not update this value. The source age limit is three times the larger +of the collection interval and `RUSTFS_OBS_METER_INTERVAL` (with its normal default). +`rustfs_storage_snapshot_max_age_seconds` exports the remaining validity budget +after deducting collection time from that limit. A slow RPC or usage read therefore +cannot make an already expired observation fresh by completing. An unavailable +source leaves the last success unchanged; a stalled collector stops exporting its +old snapshot after the source age limit. + +A Collector may cache a point after RustFS stops exporting it. The recording rules +therefore require both a fresh source and a raw point timestamp at least as new as +that source's last successful collection. This also removes cached optional fields +and old disk IDs when a new snapshot no longer contains them. The timestamp check +must run on the raw selector, before label rewriting or recording the value; +otherwise PromQL can substitute the query evaluation time for the original sample +time. The comparison uses Prometheus millisecond precision so co-published points +are not excluded by submillisecond rounding. Current recording rules preserve the +original name in `source_metric`. +When adding a storage metric, add its matching rule to that file. + +Collection and export are asynchronous, not a transaction across instruments. A +snapshot published during an export can briefly mix adjacent observations or +withhold a value; the next complete export/collection converges. Source failure visibility is bounded +by the published maximum age plus the scrape/rule intervals. Cached capacity has +its own observation age and state; a fresh collection does not make that capacity +live. Bucket/object counts retain the existing scanner update delay. + +## Queries and dashboards + +The bundled dashboard requires the storage recording rules. Select one **Storage +cluster** and one fresh **Cluster observer**. Global panels use that observer's +complete view. If it expires, the panel shows no data; select another observer. +Do not sum replicated global totals or independently take maxima/minima of fields +from different observers: they may describe different moments or partitions. + +For example, select the observed cluster raw capacity: + +```promql +rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="production-a",collection_scope="cluster",observer="node1:9000"} +``` + +For a per-drive API rate, select owner-scoped counters before `rate`, then require +a current observation. Sum the resulting rates only across the desired drives: + +```promql +sum by (rustfs_cluster_id, api) ( + rate(rustfs_system_drive_api_calls_total{rustfs_cluster_id="production-a",collection_scope="local"}[5m]) + and ignoring(source_metric) + rustfs:storage:current{source_metric="rustfs_system_drive_api_calls_total",rustfs_cluster_id="production-a",collection_scope="local"} +) +``` + +A missing series is unknown, not zero. Local totals include only fresh owners and +can decrease when a node becomes unreachable. Use a selected global observer for +configured topology and its explicit unknown/stale indicators. + +## Rolling upgrades and validation + +Install the rules, resource attribute, and dashboard together. During rolling +upgrades, old releases have no `collection_scope`; strict new-scope selectors +exclude their replicated series. Local panels initially contain only upgraded +owners. Global panels become available when an upgraded observer publishes a fresh +snapshot. Historical old-label series remain available for retrospective queries. +Do not combine old and new counter histories into one rate. + +Run the PromQL regression fixtures from `.docker/observability/tests`: + +```bash +promtool test rules storage-rules.test.yml +``` + +The native pipeline test in +`crates/e2e_test/src/storage_metric_ownership_test.rs` requires pinned Collector, +Prometheus, previous-release RustFS, and current RustFS executables. Set +`RUSTFS_OTELCOL_BINARY`, `RUSTFS_PROMETHEUS_BINARY`, +`RUSTFS_METRICS_BASELINE_BINARY`, and `CARGO_BIN_EXE_rustfs` to those files. +Optionally set `RUSTFS_METRICS_E2E_ARTIFACTS` to retain logs and Prometheus data. +Run only this external-tool test: + +```bash +cargo test --locked -p e2e_test storage_metric_ownership_pipeline -- --ignored --nocapture +``` + +The test first reproduces duplicated global details with the previous release, +rolls four nodes forward, verifies owner and disk identity, and checks node loss +and recovery while the Collector remains running. SDK unit tests cover removed +series, replacements, counter resets, invalid updates, and a stalled collection; +PromQL fixtures cover shared paths across clusters/pools and stale cached fields. diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index ceeff86cc..3cc99bc4e 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -262,9 +262,11 @@ The full schema, lease, mixed-version, retry, privacy, and metric requirements a ## Reconcile legacy transition-version metadata -This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version. +The single-record inspection routes below are implemented. GET audits every physical metadata copy, verifies the legacy tuple, and performs a bounded live version-model probe. POST validates confirmation and the complete expected tuple, repeats inspection, and returns `backend-unavailable` with `reason_code: write_fence_unavailable`, `changed: false`, and `post_ready: false` when migration is needed. Persistent backfill remains disabled until conditional per-generation `xl.meta` writes and the dedicated fleet capability are available. An already explicit, converged record can return `migrated` without changing bytes. -The approved interface is synchronous and accepts exactly one bucket/object/local-version tuple: +Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata inspection never mutates the remote tier or local metadata. + +The interface is synchronous and accepts exactly one bucket/object/local-version tuple: ```text GET /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object=&versionId= @@ -273,7 +275,7 @@ POST /rustfs/admin/v3/ilm/transition/state/reconcile?bucket=&object= "$RUN_DIR/listing.json" -rm -f target/nextest/e2e-nightly/junit.xml -set +e -cargo nextest run --profile e2e-nightly -p e2e_test -E "$FILTER" -test_exit=$? -set -e -cp target/nextest/e2e-nightly/junit.xml "$RUN_DIR/junit.xml" -scripts/python_bin.sh scripts/check_test_wiring.py --finish-scanner-heal "$RUN_DIR" "$test_exit" -scripts/python_bin.sh scripts/check_test_wiring.py --check-scanner-heal "$RUN_DIR" "$CASE" +scripts/run_scanner_heal_evidence_case.sh --case background-target-restart +scripts/run_scanner_heal_evidence_case.sh --case ec84-target-drive-restart ``` Set `RUSTFS_E2E_EXPECTED_FEATURES` to the actual intended e2e crate feature set, @@ -234,15 +232,246 @@ For automation, `--check-scanner-heal-release "$RUN_DIR"` emits one compact JSON decision and exits nonzero while blocked. `verified_cases` contains only cases that pass the complete receipt, build provenance, nextest/JUnit and real oracle checks; `rejected_cases` names registered cases that do not, and -`pending_gates` names the unimplemented release requirements. Approval requires -every registered case to verify, `pending_gates` to be empty, and a future -registry schema capable of representing the complete release matrix. Schema 1 -is deliberately marked `release_schema_capable: false`: it models only the -single-version, unversioned-object restart/crash cases and cannot represent -mixed-version, rollback, EC8+4 or performance evidence. A focused run, -synthetic harness, compile-only result, skipped/retried test, ordinary CI -success, or removal of pending text therefore cannot become a release approval. +`pending_gates` names the unimplemented release requirements and +`pending_lanes` names the structured release lanes that still need real +evidence. Schema 1 is deliberately marked `release_schema_capable: false` +because it models only the single-version, unversioned-object restart/crash +cases. Schema 2 can describe the wider release matrix, but approval still +requires every registered case to verify and every required gate to leave +`pending` only after a future checker can bind it to real feature-specific +evidence. The current checker hard-rejects missing structured requirements and +pending gates mapped to an implemented lane, so clearing pending text cannot +become approval. A focused run, synthetic harness, compile-only result, +skipped/retried test, ordinary CI success, or unregistered mixed-version, +rollback, EC8+4 or performance claim therefore cannot become a release approval. +For high-risk rollback gates, `evidence_fields` records the specific proof +fields that a future real-evidence checker must bind before a pending gate can +move out of the blocked set. G03 keeps scoped ACK tied to durable root +publication, ACK request identity, participating peer capability snapshots, and +mixed-peer fallback oracles; G09 keeps mixed-version reader, writer, and rollback +payload evidence explicit. These fields are part of the release contract, not +evidence by themselves. + +The upgrade compatibility E2E can emit raw G09 JSON artifacts when +`RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR` points at a fresh, task-owned directory. +The rolling mixed-version test writes `G09-mixed_version_reader_evidence.json` +and `G09-mixed_version_writer_evidence.json` after the old/new reader and writer +assertions pass. The bucket-metadata rollback test writes +`G09-rollback_payload_evidence.json` after the current -> previous -> current +round trip has read back the known bucket configuration and objects. These +artifacts are measured inputs for a later release bundle; the bundle must still +record their relative paths, hashes, command provenance, timestamps, roles, +participating revisions, and case lists before +`--check-scanner-heal-release-bundle` can validate them. + +For a release-candidate or PR-head Linux x86_64 host, run the full raw G09 +artifact pass with: + +```bash +scripts/run_scanner_heal_g09_upgrade_evidence.sh +``` + +The script mirrors the pinned previous-release asset used by the upgrade +workflow, builds the current checkout, runs the mixed-version and rollback E2E +lanes, and fails unless all three raw G09 artifacts are measured, revision-bound, +and role-bound. Use `--source-binary` for a custom previous-release binary on +another platform, or `--test mixed-version|rollback` while narrowing a failure. +It performs a free-space preflight before building so a saturated validation +host fails before producing partial evidence. + +Operator-collected G01 root/quota authority evidence can be packaged with: + +```bash +scripts/run_scanner_heal_authority_evidence.py \ + --root-authority-json /path/to/root-authority.json \ + --quota-authority-json /path/to/quota-authority.json \ + --out-dir /path/to/authority-descriptor +``` + +The producer rejects fixture, dry-run, synthetic, stale-revision, incomplete +root authority, and incomplete quota authority inputs before writing +`release-bundle-authority.json`. The descriptor validates only G01; it still +needs the full bundle assembler and the remaining release lanes before a release +can be approved. + +Operator-collected G03 scoped ACK evidence can be packaged with: + +```bash +scripts/python_bin.sh scripts/run_scanner_heal_scoped_ack_evidence.py \ + --proof-json /path/to/scoped-ack-proof.json \ + --out-dir /path/to/scoped-ack-descriptor +``` + +The producer requires durable root publication, exact request identity, +participating peer capability, mixed-peer fallback, and mixed-version +provenance before writing `release-bundle-scoped-ack.json`. + +The W16 recovery-intent and quota-authority lanes can emit raw G04/G12 JSON +artifacts with: + +```bash +scripts/run_scanner_heal_w16_recovery_evidence.sh +``` + +The runner builds the current checkout, runs the scanner recovery-intent and +disabled-startup crash-boundary tests, runs the scanner quota reset-preservation +tests, and runs the distributed hard-quota admission E2E. A full run writes +`release-bundle-w16.json` and validates the G04 and G12 gates with +`--check-scanner-heal-release-bundle-gate`. Use `--test g04|g12` while narrowing +a failure; a single gate descriptor still does not approve the complete release +bundle. + +The W13 durable MRF replay lanes can emit raw G07/G08/P4 JSON artifacts with: + +```bash +scripts/run_scanner_heal_w13_mrf_evidence.sh +``` + +The runner builds the current checkout, runs the ignored MRF evidence test, and +writes `release-bundle-w13.json` for `--check-scanner-heal-release-bundle-gate`. +Use `--test g07|g08|p4` while narrowing a failure. G08 disk-full evidence must +run against a real fillable filesystem: on Linux as root the runner mounts a +small tmpfs automatically, otherwise pass `--enospc-root` pointing at a +pre-mounted small filesystem. P4 is release evidence only when it completes the +default two-hour soak; `--allow-short-soak` is diagnostic and skips P4 bundle +gate validation. + +Already collected W13 raw MRF artifacts can be re-packaged and checked without +rerunning the Rust test with: + +```bash +scripts/python_bin.sh scripts/run_scanner_heal_mrf_evidence.py \ + --run-dir /path/to/scanner-heal-w13-evidence-run +``` + +The producer rejects fixture, dry-run, synthetic, stale-revision, incomplete +MRF responsibility, missing disk-full ENOSPC observations, and short P4 cleanup +soaks before writing `release-bundle-w13.json`. + +Legacy rollback evidence for R-L is assembled from a measured proof JSON: + +```bash +scripts/python_bin.sh scripts/run_scanner_heal_legacy_rollback_evidence.py \ + --proof-json /path/to/legacy-rollback-proof.json \ + --out-dir /path/to/legacy-rollback-descriptor +``` + +When the real release lanes have produced their dedicated artifacts, validate +the complete hard-gate bundle with: + +```bash +scripts/python_bin.sh scripts/check_test_wiring.py \ + --check-scanner-heal-release-bundle /path/to/release-evidence.json +``` + +For a single Linux handoff checklist that keeps the measured runners in a +stable order, generate the Scanner/Heal Linux evidence plan: + +```bash +scripts/python_bin.sh scripts/run_scanner_heal_linux_evidence_plan.py \ + --write-plan --out-dir /path/to/plan-dir +``` + +The plan is only an execution manifest. Its `evidence_type` is `plan_only`, and +it cannot satisfy any Gxx/Wxx/Rxx gate. Use `--run-preflight` only for the +lightweight registry and runner self-tests before starting a long Linux run. +After or during a Linux run, check which planned artifacts are still missing +without approving the release bundle: + +```bash +scripts/python_bin.sh scripts/run_scanner_heal_linux_evidence_plan.py \ + --status-root /path/to/run-root --format json +``` + +The status command exits nonzero while evidence is missing or malformed and +keeps `release_approved: false`; use its `pending_gates` and `next_step` fields +for issue writeback and failure triage. + +Lane descriptors can be assembled into that bundle with: + +```bash +scripts/python_bin.sh scripts/check_test_wiring.py \ + --assemble-scanner-heal-release-bundle \ + /path/to/release-bundle-g14.json \ + /path/to/release-bundle-w16.json \ + /path/to/assembled-release-bundle +``` + +The final argument is the new output directory. Every preceding argument is a +measured descriptor, and each descriptor gate must keep its own passing status, +lane identity, measured evidence type, complete required fields, relative +artifact paths, and matching SHA256 hashes. The assembler may verify only the +lanes already present; a partial assembled bundle remains blocked until all +release gates are supplied. + +The bundle checker is intentionally stricter than the case checker. It requires +schema 2 registry metadata, `evidence: measured`, the current checkout revision, +all G01-G14/P1-P4/R-E/R-D/R-L gates, per-gate `status: pass`, lane identity, +relative artifact paths, matching SHA256 hashes, and non-empty summaries. It +also binds each evidence field to its own run provenance: `source_revision`, +`run_id`, `measurement_window_id`, timezone-qualified `started_at` and +`finished_at`, command arguments, and artifact format. The field +`source_revision` must match the bundle revision, and measured performance +duration cannot exceed the recorded run window. +When an evidence or profile artifact declares a JSON format, the checker also +opens that artifact and requires its payload to repeat the same measured +`source_revision`, `run_id`, `measurement_window_id`, gate and field identity; +profile sub-artifacts must additionally name their artifact kind. Updating only +the outer bundle hash cannot turn a stale JSON summary into current release +evidence. + +The hard evidence shape remains claim-specific: mixed-version gates must name at +least two participating versions, crash/durable replay gates must include +crash-boundary evidence, G14 must record EC8+4 with at least three nodes and four +drives per node plus multi-set and multi-pool evidence, performance gates need +measured durations, P3's pressure run needs at least two hours, and P1 needs a +symbolized profile summary with resolved samples. Every G14 field and every +performance gate's fields must also share one `measurement_window_id`, so EC8+4, +multi-set/multi-pool, ABBA, throughput, and profiling artifacts cannot be +stitched together from unrelated runs. P1 `profile_evidence` must bind every +required profile artifact kind (`allocation-profile`, `flamegraph`, +`rss-samples`, and `save-frequency`) with a relative path, artifact format, +non-empty file, matching SHA256, and descriptor-level `source_revision`, +`run_id`, and `measurement_window_id` values that match the parent profile +evidence. JSON profile artifact wrappers must also carry a bundled raw profile +path with matching raw SHA256 and byte count plus the relevant allocation, RSS, +sample, or save-frequency cost counters. Missing, synthetic, stale, tampered, +undersized, cross-run, or topology-mismatched evidence returns a compact blocked +or invalid JSON result and a nonzero exit. + +The status-and-outcome raw collector normalizes live observations into the three +measured raw JSON artifacts for G05, G06, and R-D. The descriptor producer then +consumes those artifacts. The inputs must all carry schema 1, measured evidence, +matching `source_revision`, a shared `run_id`, a shared +`measurement_window_id`, matching `started_at`/`finished_at` timestamps, and +non-empty command provenance. The collector and producer reject synthetic input, +missing required cases, and command-line run/window/time overrides that would +relabel raw artifacts from another status-and-outcome run. + +The scheduler-pressure lane must also carry the numbers needed to close W09, +W10, and W11: bounded deferred item/byte/age limits, zero duplicate tasks, +pressure pacing engagement, recovery and lock-hold timings, fixed offered load, +foreground p95/p99 latency, throughput, error count, attempt-cost samples, and +completed heal object counts. + +This command validates the evidence package; it does not create evidence. A +handwritten JSON file, a synthetic harness pass, a single focused case, or a +local unit fixture still cannot satisfy the distributed, mixed-version, +crash-restart, durable MRF replay, EC8+4, ABBA, or profiling gates. Run parser/receipt regressions with `scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures validate the checker only and produce no runtime or performance evidence. + +For local bundle-shape dry runs, generate a task-owned fixture directory with: + +```bash +scripts/python_bin.sh scripts/check_test_wiring.py \ + --write-scanner-heal-release-bundle-fixture /path/to/fixture-dir +``` + +The generated file is marked `fixture_only` and is intentionally rejected by the +release bundle checker. Use it to rehearse field names, artifact paths, hashes, +profile artifact membership, mixed-version roles, and same-window provenance +before copying the shape into a real measured bundle. It is not ABBA, profile, +mixed-version, crash-restart, or release approval evidence. diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md index bde2ee748..99f6e39f9 100644 --- a/docs/testing/distributed-e2e.md +++ b/docs/testing/distributed-e2e.md @@ -77,4 +77,30 @@ cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distribut The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check. +Scanner/Heal G09 release evidence can be collected on Linux x86_64 with the +same pinned previous-release binary used by `e2e-upgrade.yml`: + +```bash +scripts/run_scanner_heal_g09_upgrade_evidence.sh +``` + +The runner builds the current `rustfs` binary, downloads and verifies the +pinned previous release, runs the mixed-version rolling-upgrade and rollback +tests, and leaves the required raw G09 artifacts under +`target/scanner-heal-g09-upgrade-evidence//`. These artifacts are +inputs for the Scanner/Heal release bundle gate; the runner does not mark the +full release matrix complete by itself. + +The distributed Scanner/Heal EC8+4 restart case is registered as +`ec84-target-drive-restart` and selected by this profile: + +```bash +scripts/run_scanner_heal_evidence_case.sh --case ec84-target-drive-restart +``` + +That command records the current build, runs exactly the registered +`distributed::heal_test` case, validates the JUnit/listing/oracle receipt, and +keeps the wider release gate blocked until the remaining release evidence lanes +have measured artifacts. + Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed ` after adding or renaming a case. diff --git a/helm/README.md b/helm/README.md index c246a3bac..38c615215 100644 --- a/helm/README.md +++ b/helm/README.md @@ -160,7 +160,7 @@ behavior. | config.rustfs.kms.vault.vault_backend | string | `""`| The vault backend, `vault-kv2` or `vault-transit`. | | config.rustfs.kms.vault.vault_address | string | `""`| The vault address. | | config.rustfs.kms.vault.vault_token | string | `""`| The vault token. Rendered into a dedicated Secret (`-kms-secret`), never into the ConfigMap. | -| config.rustfs.kms.vault.vault_mount_path | string | `"transit"`| The vault mount path, only works if `vault_backend` equals `vault-transit` . | +| config.rustfs.kms.vault.vault_mount_path | string | `"transit"`| The vault mount path. Rendered as `RUSTFS_KMS_VAULT_MOUNT_PATH` for `vault-transit`, and as `RUSTFS_KMS_VAULT_KV_MOUNT` for `vault-kv2` (only when set; unset keeps the `secret` default). | | config.rustfs.kms.vault.default_key | string | `"transit"`| The master key id for RustFS. | | extraEnv | list | `[]` | Extra environment variables for the RustFS container. An explicit `RUSTFS_LOCAL_ENDPOINT_HOST` or `RUSTFS_VOLUMES`, or a bounded, dynamic, or unrecognized startup mode, disables generated anchor injection. `POD_NAME` and `RUSTFS_ADDRESS` remain independent overrides. | | extraVolumes | list | `[]` | Extra volumes to add to the pod spec. Supported in both standalone (Deployment) and distributed (StatefulSet) modes. | diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 151099e8e..edf9de206 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -392,6 +392,7 @@ zip = { workspace = true } # Enables the shared MockWarmBackend / xl.meta assertion helpers exposed via # the ecstore `api::tier::test_util` facade module (rustfs/backlog#1148 ilm-6). rustfs-ecstore = { workspace = true, features = ["test-util"] } +rustfs-heal = { workspace = true, features = ["test-util"] } [build-dependencies] http.workspace = true diff --git a/rustfs/src/admin/handlers/health.rs b/rustfs/src/admin/handlers/health.rs index cecbd9292..57d405a74 100644 --- a/rustfs/src/admin/handlers/health.rs +++ b/rustfs/src/admin/handlers/health.rs @@ -211,6 +211,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::StorageQuorumUnavailable], + storage_details: None, }; let parts = build_health_response_parts( Method::GET, @@ -233,6 +234,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, }; let parts = build_health_response_parts( Method::GET, @@ -255,6 +257,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::StorageAndIamUnavailable], + storage_details: None, }; let parts = build_health_response_parts( Method::GET, @@ -284,6 +287,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::LockQuorumUnavailable], + storage_details: None, }; let liveness = build_health_response_parts( @@ -328,6 +332,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::StorageAndIamUnavailable], + storage_details: None, }; let parts = build_health_response_parts( Method::HEAD, @@ -398,6 +403,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, }; let parts = build_health_response_parts(Method::HEAD, HealthProbe::Readiness, Some(&report), "rustfs-endpoint", None, None); @@ -417,6 +423,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::StorageQuorumUnavailable], + storage_details: None, }; let parts = build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs-endpoint", None, None); @@ -440,6 +447,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, }; let parts = build_health_response_parts( Method::GET, diff --git a/rustfs/src/admin/handlers/ilm_transition.rs b/rustfs/src/admin/handlers/ilm_transition.rs index 87595ff8e..2a8893a73 100644 --- a/rustfs/src/admin/handlers/ilm_transition.rs +++ b/rustfs/src/admin/handlers/ilm_transition.rs @@ -15,21 +15,22 @@ use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{current_action_credentials, object_store_from_extensions}; -use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; +use crate::admin::storage_api::bucket::{is_reserved_or_invalid_bucket, utils::is_valid_object_prefix}; use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::lifecycle::{ IlmRecoveryClassification, IlmRecoveryControlView, IlmRecoveryDispositionExecutionOutcome, IlmRecoveryDispositionReasonCode, - IlmRecoveryDispositionState, IlmRecoveryExportObservation, IlmRecoveryProtocol, ManualTransitionCancelCheck, - ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink, ManualTransitionQueueSnapshot, - ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim, - TransitionOperatorDeleteResult, TransitionOperatorError, TransitionRecoveryRetryResult, TransitionRecoveryRetryStatus, - claim_manual_transition_scope_admission, create_recovery_export, delete_manual_transition_scope_admission_if_current, - delete_transition_candidate_for_operator, dry_run_recovery_disposition, enqueue_transition_for_existing_objects_scoped, - execute_recovery_disposition, finalize_missing_transition_transaction_for_operator, inspect_recovery_control, - inspect_recovery_export_observation, inspect_transition_recovery_retry_for_operator, - inspect_transition_transaction_for_operator, list_recovery_controls, load_manual_transition_job_record, - load_manual_transition_scope_admission, load_recovery_export, manual_transition_job_lease_expired, - manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired, + IlmRecoveryDispositionState, IlmRecoveryExportObservation, IlmRecoveryProtocol, LegacyTransitionStateReconcileError, + LegacyTransitionStateReconcileRequest, LegacyTransitionStateReconcileResponse, LegacyTransitionStateReconcileSelector, + ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink, + ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission, + ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError, TransitionRecoveryRetryResult, + TransitionRecoveryRetryStatus, claim_manual_transition_scope_admission, create_recovery_export, + delete_manual_transition_scope_admission_if_current, delete_transition_candidate_for_operator, dry_run_recovery_disposition, + enqueue_transition_for_existing_objects_scoped, execute_recovery_disposition, + finalize_missing_transition_transaction_for_operator, inspect_recovery_control, inspect_recovery_export_observation, + inspect_transition_recovery_retry_for_operator, inspect_transition_transaction_for_operator, list_recovery_controls, + load_manual_transition_job_record, load_manual_transition_scope_admission, load_recovery_export, + manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired, persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel, retry_transition_recovery_for_operator, save_manual_transition_job_record, update_manual_transition_job_record, @@ -267,6 +268,16 @@ pub fn register_ilm_transition_route(r: &mut S3Router) -> std::i format!("{ADMIN_PREFIX}/v3/ilm/recovery/exports/{{export_id}}").as_str(), AdminOperation(&IlmRecoveryExportDownloadHandler {}), )?; + r.insert( + Method::GET, + format!("{ADMIN_PREFIX}/v3/ilm/transition/state/reconcile").as_str(), + AdminOperation(&LegacyTransitionStateReconcileInspectHandler {}), + )?; + r.insert( + Method::POST, + format!("{ADMIN_PREFIX}/v3/ilm/transition/state/reconcile").as_str(), + AdminOperation(&LegacyTransitionStateReconcileApplyHandler {}), + )?; Ok(()) } @@ -1001,6 +1012,98 @@ fn validate_recovery_observation_receipt( Ok(receipt.observation) } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct LegacyTransitionStateReconcileQuery { + bucket: Option, + object: Option, + #[serde(rename = "versionId")] + version_id: Option, +} + +fn parse_legacy_transition_state_reconcile_query(query: Option<&str>) -> S3Result { + let query: LegacyTransitionStateReconcileQuery = serde_urlencoded::from_bytes(query.unwrap_or_default().as_bytes()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid legacy transition-state reconcile query"))?; + let bucket = query + .bucket + .filter(|bucket| !bucket.is_empty()) + .ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InvalidRequest, "bucket is required"))?; + if is_reserved_or_invalid_bucket(&bucket, false) { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidBucketName, "invalid bucket name")); + } + + let object = query + .object + .filter(|object| !object.is_empty()) + .ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InvalidRequest, "object is required"))?; + if !is_valid_object_prefix(&object) || object.contains('\n') || object.contains('\r') { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid object name")); + } + + let version_id = query + .version_id + .filter(|version_id| !version_id.is_empty()) + .ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InvalidRequest, "versionId is required"))?; + let version_id = if version_id == "null" { + version_id + } else { + let parsed = Uuid::parse_str(&version_id) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid local versionId"))?; + if parsed.is_nil() { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid local versionId")); + } + parsed.to_string() + }; + + Ok(LegacyTransitionStateReconcileSelector { + bucket, + object, + version_id, + }) +} + +fn validate_legacy_transition_state_reconcile_request( + query_selector: &LegacyTransitionStateReconcileSelector, + confirm: bool, + request_selector: &LegacyTransitionStateReconcileSelector, +) -> S3Result<()> { + if !confirm { + return Err(admin_s3_error( + AdminS3ErrorCode::InvalidRequest, + "legacy transition-state reconciliation requires confirm=true; use GET to inspect without changes", + )); + } + if request_selector != query_selector { + return Err(admin_s3_error( + AdminS3ErrorCode::InvalidRequest, + "request selector must exactly match the query selector", + )); + } + Ok(()) +} + +fn map_legacy_transition_state_reconcile_error(err: LegacyTransitionStateReconcileError) -> S3Error { + match err { + LegacyTransitionStateReconcileError::InvalidSelector(_) | LegacyTransitionStateReconcileError::InvalidRequest(_) => { + admin_s3_error(AdminS3ErrorCode::InvalidRequest, "invalid legacy transition-state reconciliation request") + } + LegacyTransitionStateReconcileError::StaleExpectedTuple(_) | LegacyTransitionStateReconcileError::Corrupt(_) => { + admin_s3_error( + AdminS3ErrorCode::OperationAborted, + "legacy transition-state reconciliation metadata is stale or corrupt", + ) + } + LegacyTransitionStateReconcileError::WriteFenceUnavailable(_) => admin_s3_error( + AdminS3ErrorCode::OperationAborted, + "legacy transition-state reconciliation could not acquire safe write authority", + ), + LegacyTransitionStateReconcileError::BackendUnavailable(_) => admin_s3_error( + AdminS3ErrorCode::InternalError, + "legacy transition-state reconciliation backend is unavailable", + ), + } +} + fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error { match err { TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"), @@ -1961,6 +2064,58 @@ impl Operation for TransitionReconcileApplyHandler { } } +pub struct LegacyTransitionStateReconcileInspectHandler {} + +#[async_trait::async_trait] +impl Operation for LegacyTransitionStateReconcileInspectHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?; + let selector = parse_legacy_transition_state_reconcile_query(req.uri.query())?; + let Some(store) = object_store_from_extensions(&req.extensions) else { + return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); + }; + let response: LegacyTransitionStateReconcileResponse = store + .inspect_legacy_transition_state(selector) + .await + .map_err(map_legacy_transition_state_reconcile_error)?; + json_response(StatusCode::OK, &response) + } +} + +pub struct LegacyTransitionStateReconcileApplyHandler {} + +#[async_trait::async_trait] +impl Operation for LegacyTransitionStateReconcileApplyHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?; + let selector = parse_legacy_transition_state_reconcile_query(req.uri.query())?; + let store = object_store_from_extensions(&req.extensions); + let mut input = req.input; + let body = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|_| { + admin_s3_error( + AdminS3ErrorCode::InvalidRequest, + "legacy transition-state reconciliation body is too large or unreadable", + ) + })?; + let request: LegacyTransitionStateReconcileRequest = serde_json::from_slice(&body).map_err(|_| { + admin_s3_error( + AdminS3ErrorCode::InvalidRequest, + "legacy transition-state reconciliation request must be valid JSON", + ) + })?; + validate_legacy_transition_state_reconcile_request(&selector, request.confirm, &request.selector)?; + let Some(store) = store else { + return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); + }; + + let response: LegacyTransitionStateReconcileResponse = store + .reconcile_legacy_transition_state(request) + .await + .map_err(map_legacy_transition_state_reconcile_error)?; + json_response(StatusCode::OK, &response) + } +} + #[cfg(test)] mod tests { use super::*; @@ -2592,12 +2747,87 @@ mod tests { let apply = src .split("impl Operation for TransitionReconcileApplyHandler") .nth(1) - .and_then(|block| block.split("#[cfg(test)]").next()) + .and_then(|block| block.split("pub struct LegacyTransitionStateReconcileInspectHandler").next()) .expect("apply handler block"); assert!(apply.contains("AdminAction::SetTierAction")); assert!(!apply.contains("AdminAction::ListTierAction")); } + #[test] + fn legacy_transition_state_query_requires_one_exact_selector() { + let version_id = Uuid::new_v4(); + let query = format!("bucket=test-bucket&object=logs%2F2026%20report&versionId={version_id}"); + let selector = parse_legacy_transition_state_reconcile_query(Some(&query)).expect("exact version selector should parse"); + assert_eq!(selector.bucket, "test-bucket"); + assert_eq!(selector.object, "logs/2026 report"); + assert_eq!(selector.version_id, version_id.to_string()); + + let unversioned = + parse_legacy_transition_state_reconcile_query(Some("bucket=test-bucket&object=logs%2Fcurrent&versionId=null")) + .expect("explicit null selector should parse"); + assert_eq!(unversioned.version_id, "null"); + + for query in [ + None, + Some("bucket=test-bucket&object=key"), + Some("bucket=test-bucket&object=&versionId=null"), + Some("bucket=test-bucket&object=key&versionId="), + Some("bucket=test-bucket&object=key&versionId=not-a-uuid"), + Some("bucket=test-bucket&object=key&versionId=00000000-0000-0000-0000-000000000000"), + Some("bucket=test-bucket&object=bad%0Akey&versionId=null"), + Some("bucket=test-bucket&object=key&versionId=null&prefix=wide"), + Some("bucket=test-bucket&bucket=other-bucket&object=key&versionId=null"), + ] { + assert!( + parse_legacy_transition_state_reconcile_query(query).is_err(), + "query should fail closed: {query:?}" + ); + } + } + + #[test] + fn legacy_transition_state_apply_requires_confirmation_and_matching_selector() { + let selector = LegacyTransitionStateReconcileSelector { + bucket: "test-bucket".to_string(), + object: "key".to_string(), + version_id: "null".to_string(), + }; + let different = LegacyTransitionStateReconcileSelector { + object: "other-key".to_string(), + ..selector.clone() + }; + + assert!(validate_legacy_transition_state_reconcile_request(&selector, false, &selector).is_err()); + assert!(validate_legacy_transition_state_reconcile_request(&selector, true, &different).is_err()); + validate_legacy_transition_state_reconcile_request(&selector, true, &selector) + .expect("confirmed exact selector should pass handler validation"); + } + + #[test] + fn legacy_transition_state_routes_use_read_and_write_tier_actions() { + let src = include_str!("ilm_transition.rs"); + let inspect = src + .split("impl Operation for LegacyTransitionStateReconcileInspectHandler") + .nth(1) + .and_then(|block| { + block + .split("impl Operation for LegacyTransitionStateReconcileApplyHandler") + .next() + }) + .expect("legacy inspect handler block"); + assert!(inspect.contains("AdminAction::ListTierAction")); + assert!(!inspect.contains("AdminAction::SetTierAction")); + + let apply = src + .split("impl Operation for LegacyTransitionStateReconcileApplyHandler") + .nth(1) + .and_then(|block| block.split("#[cfg(test)]").next()) + .expect("legacy apply handler block"); + assert!(apply.contains("AdminAction::SetTierAction")); + assert!(!apply.contains("AdminAction::ListTierAction")); + assert!(apply.contains("validate_legacy_transition_state_reconcile_request")); + } + #[test] fn manual_transition_query_defaults_to_bounded_run() { let (bucket, options, run_mode) = @@ -3066,6 +3296,24 @@ mod tests { assert_eq!(cancel_err.message(), Some("authentication required")); } + #[tokio::test] + async fn legacy_transition_state_handlers_reject_missing_credentials_before_selector_or_body() { + let path = "/rustfs/admin/v3/ilm/transition/state/reconcile?bucket=test-bucket&object=key&versionId=null"; + let inspect_err = LegacyTransitionStateReconcileInspectHandler {} + .call(credential_less_admin_request(Method::GET, path), Params::new()) + .await + .expect_err("inspect handler must reject unsigned requests"); + assert_eq!(inspect_err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(inspect_err.message(), Some("authentication required")); + + let apply_err = LegacyTransitionStateReconcileApplyHandler {} + .call(credential_less_admin_request(Method::POST, path), Params::new()) + .await + .expect_err("apply handler must reject unsigned requests"); + assert_eq!(apply_err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(apply_err.message(), Some("authentication required")); + } + #[test] fn manual_transition_job_handlers_authorize_validate_and_load_store() { let src = include_str!("ilm_transition.rs"); diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index e6795566c..b35fd4c89 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -23,6 +23,7 @@ use crate::admin::runtime_sources::{ current_or_init_kms_runtime_service_manager, }; use crate::admin::storage_api::config::{read_admin_config, save_admin_config}; +use crate::admin::storage_api::ecstore_topology::is_dist_erasure; use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::runtime::ECStore; use crate::admin::storage_api::s3::{S3ErrorCode, error as admin_s3_error}; @@ -507,6 +508,21 @@ pub async fn reload_persisted_kms_config() -> Result<(), String> { reload_persisted_kms_config_from_store(store, kms_service_manager_from_context(), "peer_reload").await } +/// Whether a reload may return early because this node is already serving +/// exactly the persisted configuration. +/// +/// Byte-identical configuration is not sufficient on its own. A node whose KMS +/// failed to start keeps its configuration and sits in `Error`, so comparing +/// only the bytes turned the documented recovery call +/// (`POST /rustfs/admin/v3/kms/reload`) into a no-op that reported success and +/// left the node down — including on every peer, which reaches this same +/// function through the reload broadcast (backlog#2369 P1). Any state other +/// than `Running` falls through to `reconfigure`, which starts the service when +/// none is running. +fn kms_reload_is_already_current(status: rustfs_kms::KmsServiceStatus, config_is_unchanged: bool) -> bool { + matches!(status, rustfs_kms::KmsServiceStatus::Running) && config_is_unchanged +} + async fn reload_persisted_kms_config_from_store( store: Arc, service_manager: Arc, @@ -525,11 +541,11 @@ async fn reload_persisted_kms_config_from_store( return Err("no persisted KMS configuration is available".to_string()); }; - if service_manager + let config_is_unchanged = service_manager .get_config() .await - .is_some_and(|current| kms_config_is_unchanged(¤t, &config)) - { + .is_some_and(|current| kms_config_is_unchanged(¤t, &config)); + if kms_reload_is_already_current(service_manager.get_status().await, config_is_unchanged) { info!( event = "kms_service_state", component = LOG_COMPONENT_ADMIN, @@ -624,6 +640,51 @@ fn local_success_with_peer_report(message: &str, unconverged: &[String]) -> (boo ) } +/// What a node-local KMS backend means for a multi-node deployment +/// (backlog#2369 P7.4). +/// +/// The Local backend keeps key material on each node's own disk and generates +/// its KDF salt per node, so two nodes derive different keys from the same +/// `master_key`. An object encrypted on node A cannot be decrypted on node B: +/// behind a load balancer that shows up as intermittent 500s on reads that +/// worked a moment earlier. The product decision to warn rather than refuse +/// stands; the generic "development only" warning simply never said what +/// actually goes wrong, so an operator had no way to connect the symptom to +/// the cause. +/// +/// Returns the sentence to append to the configure response, or `None` when the +/// combination does not apply. +async fn node_local_backend_warning(backend: &rustfs_kms::KmsBackend) -> Option<&'static str> { + if !matches!(backend, rustfs_kms::KmsBackend::Local) || !is_dist_erasure().await { + return None; + } + + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_node_local_backend_in_distributed_deployment", + backend = rustfs_kms::KmsBackend::Local.as_str(), + "The Local KMS backend stores key material on each node's own disk with a per-node salt, so objects \ + encrypted on one node cannot be decrypted on another. In a distributed deployment this surfaces as \ + intermittent 500s on reads behind a load balancer. Use Vault Transit, Vault KV2 or AWS KMS for a \ + multi-node deployment" + ); + + Some( + "Warning: the Local KMS backend is node-local. Key material and its salt live on each node's own disk, so \ + objects encrypted on one node cannot be decrypted on another and reads behind a load balancer will fail \ + intermittently. Use Vault Transit, Vault KV2 or AWS KMS for a distributed deployment", + ) +} + +/// Append the node-local backend warning to a successful configure message. +fn with_node_local_backend_warning(message: String, warning: Option<&'static str>) -> String { + match warning { + Some(warning) => format!("{message}. {warning}"), + None => message, + } +} + pub fn register_kms_dynamic_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( Method::POST, @@ -746,6 +807,7 @@ impl Operation for ConfigureKmsHandler { let kms_config = configure_request.to_kms_config(); let persisted_config = kms_config.clone(); + let node_local_warning = node_local_backend_warning(&kms_config.backend).await; let (success, message, status) = match service_manager .configure_with_persistence(kms_config, || async move { save_kms_config(&persisted_config) @@ -768,7 +830,7 @@ impl Operation for ConfigureKmsHandler { let unconverged = broadcast_kms_config_reload().await; let (success, message) = local_success_with_peer_report("KMS configured successfully", &unconverged); audit.finish(KmsAdminOperation::Configure, None, None); - (success, message, status) + (success, with_node_local_backend_warning(message, node_local_warning), status) } Err(e) => { let error_msg = format!("Failed to configure KMS: {e}"); @@ -1345,6 +1407,7 @@ impl Operation for ReconfigureKmsHandler { let kms_config = configure_request.to_kms_config(); let persisted_config = kms_config.clone(); + let node_local_warning = node_local_backend_warning(&kms_config.backend).await; let (success, message, status) = match service_manager .reconfigure_with_persistence(kms_config, || async move { save_kms_config(&persisted_config) @@ -1368,7 +1431,7 @@ impl Operation for ReconfigureKmsHandler { let (success, message) = local_success_with_peer_report("KMS reconfigured and restarted successfully", &unconverged); audit.finish(KmsAdminOperation::Reconfigure, None, None); - (success, message, status) + (success, with_node_local_backend_warning(message, node_local_warning), status) } Err(e) => { let error_msg = format!("Failed to reconfigure KMS: {e}"); @@ -1420,9 +1483,10 @@ impl Operation for ReconfigureKmsHandler { mod tests { use super::{ KmsConfigLoadError, decode_persisted_kms_config, ensure_kms_config_persistable, ensure_kms_request_persistable, - kms_config_fingerprint, kms_config_is_unchanged, kms_configure_actions, kms_service_control_actions, - load_kms_config_with, local_success_with_peer_report, normalize_configure_request_secrets, open_persisted_kms_config, - redacted_canonical_config, register_kms_dynamic_route, seal_persisted_kms_config, + kms_config_fingerprint, kms_config_is_unchanged, kms_configure_actions, kms_reload_is_already_current, + kms_service_control_actions, load_kms_config_with, local_success_with_peer_report, normalize_configure_request_secrets, + open_persisted_kms_config, redacted_canonical_config, register_kms_dynamic_route, seal_persisted_kms_config, + with_node_local_backend_warning, }; use crate::admin::router::{AdminOperation, S3Router}; use crate::admin::storage_api::error::StorageError; @@ -1432,6 +1496,49 @@ mod tests { use std::path::PathBuf; use tempfile::TempDir; + /// backlog#2369 P1: a node whose KMS failed to start keeps its persisted + /// configuration, so an unchanged-bytes comparison made the documented + /// recovery call a no-op that still reported success. + #[test] + fn kms_reload_only_short_circuits_for_a_running_service() { + use rustfs_kms::KmsServiceStatus; + + assert!( + kms_reload_is_already_current(KmsServiceStatus::Running, true), + "a running service on identical configuration has nothing to apply" + ); + assert!( + !kms_reload_is_already_current(KmsServiceStatus::Running, false), + "changed configuration must always be applied" + ); + + for status in [ + KmsServiceStatus::NotConfigured, + KmsServiceStatus::Configured, + KmsServiceStatus::Error("vault unreachable at startup".to_string()), + ] { + assert!( + !kms_reload_is_already_current(status.clone(), true), + "reload must reconfigure instead of reporting success from {status:?}" + ); + } + } + + /// backlog#2369 P7.4: the operator has to learn the consequence from the + /// response, not just from a log line the configuring client never sees. + #[test] + fn a_node_local_backend_warning_reaches_the_configure_response() { + let plain = with_node_local_backend_warning("KMS configured successfully".to_string(), None); + assert_eq!(plain, "KMS configured successfully"); + + let warned = with_node_local_backend_warning( + "KMS configured successfully".to_string(), + Some("Warning: the Local KMS backend is node-local"), + ); + assert!(warned.starts_with("KMS configured successfully."), "{warned}"); + assert!(warned.contains("node-local"), "{warned}"); + } + fn assert_has_action(actions: &[Action], action: Action) { assert!(actions.contains(&action), "expected action list to contain {action:?}"); } diff --git a/rustfs/src/admin/handlers/scanner.rs b/rustfs/src/admin/handlers/scanner.rs index f96f71406..bb2e83d44 100644 --- a/rustfs/src/admin/handlers/scanner.rs +++ b/rustfs/src/admin/handlers/scanner.rs @@ -75,6 +75,10 @@ struct ScannerRecoveryIntentResponse { mode: String, intent_id: String, state: String, + actor_sha256: String, + idempotency_key_sha256: String, + request_sha256: String, + accepted_at_unix_secs: u64, } #[derive(Debug, Serialize)] @@ -332,6 +336,10 @@ fn scanner_recovery_intent_record_response( mode: record.mode, intent_id: record.intent_id, state: record.state, + actor_sha256: record.actor_sha256, + idempotency_key_sha256: record.idempotency_key_sha256, + request_sha256: record.request_sha256, + accepted_at_unix_secs: record.accepted_at_unix_secs, }; let body = serde_json::to_vec(&response).map_err(|err| { S3Error::with_message( @@ -371,10 +379,7 @@ fn scanner_recovery_intent_accept_response( fn scanner_recovery_intent_executor_id(result: &rustfs_scanner::ScannerRecoveryIntentAcceptResult) -> Option { match result { - rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record } - | rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed { record } - if matches!(record.state.as_str(), "accepted" | "running") => - { + rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record } if record.state == "accepted" => { Some(record.intent_id.clone()) } _ => None, @@ -673,7 +678,7 @@ mod tests { } #[test] - fn scanner_recovery_intent_executor_only_starts_non_terminal_work() { + fn scanner_recovery_intent_executor_starts_only_newly_accepted_work() { let mut record = rustfs_scanner::ScannerRecoveryIntentRecord { schema_version: 1, intent_id: "0".repeat(64), @@ -699,7 +704,16 @@ mod tests { record: record.clone(), }) .as_deref(), - Some(record.intent_id.as_str()) + None, + "a lost-response retry must not start a duplicate executor" + ); + record.state = "accepted".to_string(); + assert!( + scanner_recovery_intent_executor_id(&rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed { + record: record.clone(), + }) + .is_none(), + "replayed accepted records remain durable for startup/control recovery instead of duplicating work" ); record.state = "completed".to_string(); assert!( diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index aebf92849..7a6137c21 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -550,6 +550,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ SET_TIER, RouteRiskLevel::High, ), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/ilm/transition/state/reconcile", + LIST_TIER, + RouteRiskLevel::High, + ), + admin( + HttpMethod::Post, + "/rustfs/admin/v3/ilm/transition/state/reconcile", + SET_TIER, + RouteRiskLevel::High, + ), admin( HttpMethod::Get, "/rustfs/admin/v3/audit/target/list", @@ -2212,12 +2224,16 @@ mod tests { assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER); assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER); assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER); + assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/state/reconcile", LIST_TIER); + assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/state/reconcile", SET_TIER); assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SERVER_INFO); assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/records", SERVER_INFO); assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO); assert_not_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO); assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER); assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER); + assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/state/reconcile", SET_TIER); + assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/state/reconcile", LIST_TIER); } #[test] diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 5a1e66ced..1673a00a9 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -241,6 +241,8 @@ fn expected_admin_route_matrix() -> Vec { "/v3/ilm/transition/reconcile/{transaction_id}", "/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111", ), + admin_route(Method::GET, "/v3/ilm/transition/state/reconcile"), + admin_route(Method::POST, "/v3/ilm/transition/state/reconcile"), admin_route_sample( Method::DELETE, "/v3/ilm/transition/jobs/{job_id}", @@ -993,6 +995,8 @@ fn test_register_routes_cover_representative_admin_paths() { Method::POST, &admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"), ); + assert_route(&router, Method::GET, &admin_path("/v3/ilm/transition/state/reconcile")); + assert_route(&router, Method::POST, &admin_path("/v3/ilm/transition/state/reconcile")); assert_route(&router, Method::GET, &table_catalog_path("/config")); assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics")); diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index f257a9ce5..0ee0f9099 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -41,6 +41,12 @@ pub(crate) mod ecstore_cluster { }; } +/// Deployment topology. The KMS configure path needs it to tell an operator +/// that a node-local backend cannot serve a multi-node deployment. +pub(crate) mod ecstore_topology { + pub(crate) use crate::storage::storage_api::is_dist_erasure; +} + mod ecstore_config { pub(crate) use crate::storage::storage_api::ecstore_config::{com, init, storageclass}; } @@ -225,6 +231,10 @@ pub(crate) mod lifecycle { renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record, }; + pub(crate) use crate::storage::storage_api::ecstore_bucket::lifecycle::legacy_transition_state_reconcile::{ + LegacyTransitionStateReconcileError, LegacyTransitionStateReconcileRequest, LegacyTransitionStateReconcileResponse, + LegacyTransitionStateReconcileSelector, + }; pub(crate) type ManualTransitionCancelCheck = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionCancelCheck; pub(crate) type ManualTransitionQueueSnapshot = diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index a4f6a1dc5..3ba2d330e 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -775,6 +775,7 @@ mod tests { scan_plan_digest: Some([1; 32]), complete: true, tombstone: false, + segment_invalidation_proof: None, }]; DefaultAdminUsecase::narrow_data_usage_snapshot_to_measured_buckets(&mut info, ["bucket-a".to_string()]); assert_eq!(info.usage_snapshot_converged, Some(false)); diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index d77531e1f..f9c4879e3 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1389,7 +1389,10 @@ impl DefaultBucketUsecase { counter!("rustfs_create_bucket_total").increment(1); let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer( + &bucket, + rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata, + ); result } @@ -1781,7 +1784,10 @@ impl DefaultBucketUsecase { warn!(bucket = %bucket, error = ?err, "site replication bucket tagging delete hook failed"); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer( + &bucket, + rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata, + ); Ok(S3Response::new(DeleteBucketTaggingOutput {})) } @@ -2698,7 +2704,10 @@ impl DefaultBucketUsecase { warn!(bucket = %bucket, error = ?err, "site replication bucket tagging hook failed"); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer( + &bucket, + rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata, + ); Ok(S3Response::new(PutBucketTaggingOutput::default())) } @@ -2733,7 +2742,10 @@ impl DefaultBucketUsecase { warn!(bucket = %bucket, error = ?err, "site replication bucket versioning hook failed"); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer( + &bucket, + rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata, + ); Ok(S3Response::new(PutBucketVersioningOutput {})) } diff --git a/rustfs/src/app/delete_objects_stat_gating_test.rs b/rustfs/src/app/delete_objects_stat_gating_test.rs index 16096df50..701d9c531 100644 --- a/rustfs/src/app/delete_objects_stat_gating_test.rs +++ b/rustfs/src/app/delete_objects_stat_gating_test.rs @@ -25,7 +25,7 @@ //! deletes still inspect legacy or corrupt explicit Object Lock metadata even //! when the bucket configuration is confirmed absent. -use super::gating_test_env::shared_gating_ecstore; +use super::gating_test_env::{run_large_stack_test, shared_gating_ecstore}; use super::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions}; use super::storage_api::test::contract::object::{ObjectIO as _, ObjectOperations as _}; use super::storage_api::test::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; @@ -351,9 +351,15 @@ async fn malformed_persisted_retention_metadata_blocks_version_delete() { .expect("malformed retained object must survive the rejected delete"); } -#[tokio::test] +#[test] #[serial] -async fn recursive_force_delete_remains_allowed_for_plain_bucket() { +fn recursive_force_delete_remains_allowed_for_plain_bucket() { + run_large_stack_test("recursive-force-delete-plain-bucket", || async { + recursive_force_delete_remains_allowed_for_plain_bucket_case().await; + }); +} + +async fn recursive_force_delete_remains_allowed_for_plain_bucket_case() { let ecstore = shared_gating_ecstore().await; let bucket = format!("force-delete-plain-{}", Uuid::new_v4()); diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 00811cdad..0d442a48a 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -532,7 +532,11 @@ impl DefaultMultipartUsecase { .await { Ok(_) => { - rustfs_scanner::record_dirty_usage_object(&bucket, &key); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload, + ); Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })) } Err(err) => { @@ -802,7 +806,11 @@ impl DefaultMultipartUsecase { schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await; } - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::CompleteMultipartUpload, + ); Ok::<_, S3Error>(obj_info) } }); diff --git a/rustfs/src/app/object/copy.rs b/rustfs/src/app/object/copy.rs index 76192ca9d..10aa663ce 100644 --- a/rustfs/src/app/object/copy.rs +++ b/rustfs/src/app/object/copy.rs @@ -814,7 +814,11 @@ impl DefaultObjectUsecase { } } - rustfs_scanner::record_dirty_usage_object(&bucket, &key); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject, + ); Ok::<_, S3Error>((oi, dest_versioned)) } }); diff --git a/rustfs/src/app/object/delete.rs b/rustfs/src/app/object/delete.rs index eb19768ae..8f4b3c4a7 100644 --- a/rustfs/src/app/object/delete.rs +++ b/rustfs/src/app/object/delete.rs @@ -797,6 +797,18 @@ impl DefaultObjectUsecase { let resp_elements = build_event_resp_elements(&S3Response::new(DeleteObjectsOutput::default()), &request_context.request_id); let deleted_any = delete_results.iter().any(|result| result.delete_object.is_some()); + let delete_producers = delete_results + .iter() + .filter_map(|result| { + result.delete_object.as_ref().map(|deleted_object| { + if deleted_object.delete_marker && result.requested_version_id.is_none() { + rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteMarker + } else { + rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteObject + } + }) + }) + .collect::>(); let notify_bucket = bucket.clone(); spawn_background_with_context(Some(request_context), async move { let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify); @@ -838,7 +850,7 @@ impl DefaultObjectUsecase { let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); if deleted_any { - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producers(&bucket, delete_producers); } // Record write operation for capacity management (inline to avoid per-request tokio::spawn overhead) let manager = get_capacity_manager(); @@ -1101,7 +1113,10 @@ impl DefaultObjectUsecase { let manager = get_capacity_manager(); manager.record_write_operation().await; let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer( + &bucket, + rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteObject, + ); return result; } @@ -1175,7 +1190,12 @@ impl DefaultObjectUsecase { let manager = get_capacity_manager(); manager.record_write_operation().await; let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_object(&bucket, &key); + let producer = if delete_marker && version_id_clone.is_none() { + rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteMarker + } else { + rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteObject + }; + rustfs_scanner::record_dirty_usage_object_from_producer(&bucket, &key, producer); result } } @@ -1191,6 +1211,215 @@ mod tests { }; use std::sync::Arc; + #[test] + #[serial_test::serial] + fn execute_read_delete_marker_headers_in_single_and_multi_pool() { + crate::app::gating_test_env::run_large_stack_test("read-marker-headers", || async { + use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + let single_pool = crate::app::gating_test_env::shared_gating_ecstore().await; + if current_app_context().is_none() { + crate::app::runtime_sources::install_test_app_context(Arc::clone(&single_pool)).await; + } + let ambient = current_app_context().expect("read API test context"); + let (_temp_dir, _disk_paths, multi_pool) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + for (pool_count, store) in [(1, single_pool), (2, multi_pool)] { + let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms())); + let usecase = DefaultObjectUsecase::with_context(Some(context)); + for suspended in [false, true] { + let bucket = format!("read-marker-headers-{pool_count}-{}", Uuid::new_v4()); + store + .make_bucket( + &bucket, + &MakeBucketOptions { + versioning_enabled: true, + ..Default::default() + }, + ) + .await + .expect("create versioned read fixture"); + let key = "history"; + let payload = b"historical payload behind the marker"; + let original = store + .put_object( + &bucket, + key, + &mut PutObjReader::from_vec(payload.to_vec()), + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write historical version"); + if suspended { + store + .update_bucket_metadata_config( + &bucket, + crate::app::storage_api::test::bucket::metadata::BUCKET_VERSIONING_CONFIG, + b"Suspended".to_vec(), + ) + .await + .expect("suspend versioning before creating the null marker"); + } + // get_opts still reads versioning through the ambient metadata + // facade. Publish the fixture's actual config there while the + // request-bound context keeps object I/O on the selected store. + let metadata = store.get_bucket_metadata(&bucket).await.expect("fixture bucket metadata"); + crate::app::storage_api::test::set_bucket_metadata(bucket.clone(), (*metadata).clone()) + .await + .expect("publish fixture versioning config"); + let read_opts = get_opts(&bucket, key, None, None, &HeaderMap::new()) + .await + .expect("read fixture versioning config"); + assert_eq!(read_opts.versioned, !suspended); + assert_eq!(read_opts.version_suspended, suspended); + let marker = store + .delete_object( + &bucket, + key, + ObjectOptions { + versioned: !suspended, + version_suspended: suspended, + ..Default::default() + }, + ) + .await + .expect("create latest delete marker"); + let marker_id = delete_response_version_id(marker.version_id, false).expect("marker identity"); + assert_eq!(marker_id == "null", suspended); + + for explicit in [false, true] { + let requested_version = explicit.then(|| marker_id.clone()); + let get = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .version_id(requested_version.clone()) + .build() + .expect("GET request"); + let head = HeadObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .version_id(requested_version) + .build() + .expect("HEAD request"); + let get_error = usecase + .execute_get_object(build_request(get, Method::GET)) + .await + .expect_err("marker GET"); + let head_error = usecase + .execute_head_object(build_request(head, Method::HEAD)) + .await + .expect_err("marker HEAD"); + for (method, error) in [("GET", get_error), ("HEAD", head_error)] { + assert_eq!( + error.code(), + if explicit { + &S3ErrorCode::MethodNotAllowed + } else { + &S3ErrorCode::NoSuchKey + } + ); + let headers = error + .headers() + .unwrap_or_else(|| panic!("{method} marker response omitted headers")); + assert_eq!(headers.get("x-amz-delete-marker").expect("marker header"), "true"); + assert_eq!(headers.get("x-amz-version-id").expect("version header"), marker_id.as_str()); + if explicit { + let modified = marker + .mod_time + .expect("marker timestamp") + .format(&RFC1123) + .expect("HTTP date"); + assert_eq!(headers.get("last-modified").expect("marker timestamp header"), modified.as_str()); + } + let wire = error.to_http_response().expect("serialize marker error response"); + assert_eq!( + wire.status(), + if explicit { + StatusCode::METHOD_NOT_ALLOWED + } else { + StatusCode::NOT_FOUND + } + ); + assert_eq!(wire.headers()["x-amz-delete-marker"], "true"); + assert_eq!(wire.headers()["x-amz-version-id"], marker_id.as_str()); + assert_eq!(wire.headers()[http::header::CONTENT_TYPE], "application/xml"); + } + } + + let original_id = original.version_id.expect("historical version id").to_string(); + let get = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .version_id(Some(original_id.clone())) + .build() + .expect("historical GET request"); + let response = usecase + .execute_get_object(build_request(get, Method::GET)) + .await + .expect("historical GET"); + assert!(!response.headers.contains_key("x-amz-delete-marker")); + let mut body = response.output.body.expect("historical body"); + let mut actual = Vec::new(); + while let Some(chunk) = body.next().await { + actual.extend_from_slice(&chunk.expect("historical data remains readable")); + } + assert_eq!(actual, payload); + let head = HeadObjectInput::builder() + .bucket(bucket.clone()) + .key(key.to_string()) + .version_id(Some(original_id.clone())) + .build() + .expect("historical HEAD request"); + let response = usecase + .execute_head_object(build_request(head, Method::HEAD)) + .await + .expect("historical HEAD"); + assert_eq!(response.output.content_length, Some(i64::try_from(payload.len()).unwrap())); + assert_eq!(response.output.version_id.as_deref(), Some(original_id.as_str())); + assert!(!response.headers.contains_key("x-amz-delete-marker")); + + for (absent_key, version) in [("missing", None), (key, Some(Uuid::new_v4().to_string()))] { + let get = GetObjectInput::builder() + .bucket(bucket.clone()) + .key(absent_key.to_string()) + .version_id(version.clone()) + .build() + .expect("missing GET request"); + let head = HeadObjectInput::builder() + .bucket(bucket.clone()) + .key(absent_key.to_string()) + .version_id(version) + .build() + .expect("missing HEAD request"); + let get_error = usecase + .execute_get_object(build_request(get, Method::GET)) + .await + .expect_err("missing GET"); + let head_error = usecase + .execute_head_object(build_request(head, Method::HEAD)) + .await + .expect_err("missing HEAD"); + for error in [get_error, head_error] { + assert_eq!(error.code().status_code(), Some(StatusCode::NOT_FOUND)); + assert!( + error + .headers() + .is_none_or(|headers| !headers.contains_key("x-amz-delete-marker")) + ); + assert!( + error + .headers() + .is_none_or(|headers| !headers.contains_key("x-amz-version-id")) + ); + } + } + } + } + }); + } + #[test] #[serial_test::serial] fn execute_delete_marker_versions_in_single_and_multi_pool() { diff --git a/rustfs/src/app/object/extract.rs b/rustfs/src/app/object/extract.rs index 2db042fa3..ae79af6af 100644 --- a/rustfs/src/app/object/extract.rs +++ b/rustfs/src/app/object/extract.rs @@ -627,6 +627,7 @@ struct ExtractPreparedMember { write_plan: WritePlan, opts: ObjectOptions, replication: ReplicateDecision, + producer_identity: rustfs_scanner::SegmentInvalidationProducerIdentity, staging_permit: OwnedSemaphorePermit, member_permit: OwnedSemaphorePermit, } @@ -765,6 +766,7 @@ where write_plan, opts, replication, + producer_identity, staging_permit, member_permit, } = member; @@ -793,7 +795,7 @@ where // the scanner before its post-store awaits, then retains the lifecycle slot // through quota, cache, replication, and event construction. if !context.wrote_any_entry.swap(true, Ordering::AcqRel) { - rustfs_scanner::record_dirty_usage_bucket(&context.bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer(&context.bucket, producer_identity); } let success = complete_extract_member_post_commit(context, key, opts, replication, obj_info, backfilled_old_current_size).await; @@ -2042,7 +2044,7 @@ impl DefaultObjectUsecase { bucket_sse_config.as_ref().map(|(config, _timestamp)| config), original_sse, ssekms_key_id, - false, + sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some(), ); if effective_sse .as_ref() @@ -2603,6 +2605,7 @@ impl DefaultObjectUsecase { write_plan, opts, replication, + producer_identity: rustfs_scanner::SegmentInvalidationProducerIdentity::DirectoryObject, staging_permit, member_permit, }, @@ -2627,6 +2630,7 @@ impl DefaultObjectUsecase { write_plan, opts, replication, + producer_identity: rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject, staging_permit, member_permit, }, @@ -2656,6 +2660,11 @@ impl DefaultObjectUsecase { write_plan, opts, replication, + producer_identity: if is_dir { + rustfs_scanner::SegmentInvalidationProducerIdentity::DirectoryObject + } else { + rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject + }, staging_permit, member_permit, }); diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index 94be270fb..34814c33c 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -4093,6 +4093,7 @@ impl DefaultObjectUsecase { let prepared_read = match prepared_read { Ok(prepared_read) => prepared_read, Err(err) => { + let err = enrich_delete_marker_read_error(&store, &bucket, &key, &opts, err).await; lifecycle.finish_err(); return Self::complete_get_object_error(helper.version_id(version_id_for_event), err); } diff --git a/rustfs/src/app/object/head.rs b/rustfs/src/app/object/head.rs index f20eb895a..b649d1630 100644 --- a/rustfs/src/app/object/head.rs +++ b/rustfs/src/app/object/head.rs @@ -381,10 +381,17 @@ impl DefaultObjectUsecase { { return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await; } - return Err(S3Error::new(S3ErrorCode::NoSuchKey)); + return Err(enrich_delete_marker_read_error( + &store, + &bucket, + &key, + &opts, + S3Error::new(S3ErrorCode::NoSuchKey), + ) + .await); } // Other errors, such as insufficient permissions, still return the original error - return Err(ApiError::from(err).into()); + return Err(enrich_delete_marker_read_error(&store, &bucket, &key, &opts, ApiError::from(err).into()).await); } }; if info.delete_marker { @@ -396,9 +403,13 @@ impl DefaultObjectUsecase { { return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await; } - return Err(S3Error::new(S3ErrorCode::NoSuchKey)); + return Err(with_delete_marker_read_headers(S3Error::new(S3ErrorCode::NoSuchKey), &info, None)); } - return Err(S3Error::new(S3ErrorCode::MethodNotAllowed)); + return Err(with_delete_marker_read_headers( + S3Error::new(S3ErrorCode::MethodNotAllowed), + &info, + opts.version_id.as_deref(), + )); } if let Some(match_etag) = if_none_match && let Some(strong_etag) = match_etag.into_etag() diff --git a/rustfs/src/app/object/internal_put.rs b/rustfs/src/app/object/internal_put.rs index 7f24c6e5c..e932a9cd0 100644 --- a/rustfs/src/app/object/internal_put.rs +++ b/rustfs/src/app/object/internal_put.rs @@ -689,7 +689,11 @@ impl DefaultObjectUsecase { schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await; } - rustfs_scanner::record_dirty_usage_object(&bucket, &key); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::CompleteMultipartUpload, + ); Ok::<_, ApiError>(obj_info) } }); @@ -729,7 +733,11 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; - rustfs_scanner::record_dirty_usage_bucket(bucket); + rustfs_scanner::record_dirty_usage_object_from_producer( + bucket, + key, + rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload, + ); Ok(()) } } diff --git a/rustfs/src/app/object/put.rs b/rustfs/src/app/object/put.rs index f7672369a..322a4309a 100644 --- a/rustfs/src/app/object/put.rs +++ b/rustfs/src/app/object/put.rs @@ -1515,7 +1515,7 @@ impl DefaultObjectUsecase { bucket_sse_config.as_ref().map(|(config, _timestamp)| config), server_side_encryption, ssekms_key_id, - false, + sse_customer_algorithm.is_some() || sse_customer_key.is_some() || sse_customer_key_md5.is_some(), ); debug!( target: "rustfs::app::object_usecase", @@ -2054,7 +2054,11 @@ impl DefaultObjectUsecase { schedule_object_replication(obj_info.clone(), store, dsc).await; } - rustfs_scanner::record_dirty_usage_object(&bucket, &key); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject, + ); rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start); let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now); @@ -3520,6 +3524,104 @@ mod tests { ); } + async fn install_bucket_default_sse_for_test(bucket: &str, algorithm: &'static str, kms_key_id: Option<&str>) { + use crate::app::storage_api::test::bucket::utils::serialize; + use crate::app::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata}; + + let sys = get_global_bucket_metadata_sys().expect("bucket metadata system"); + let metadata = { + let sys = sys.read().await; + sys.get(bucket).await.expect("bucket metadata cached") + }; + let mut metadata = (*metadata).clone(); + let config = ServerSideEncryptionConfiguration { + rules: vec![ServerSideEncryptionRule { + apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault { + sse_algorithm: ServerSideEncryption::from_static(algorithm), + kms_master_key_id: kms_key_id.map(|id| id.to_string()), + }), + blocked_encryption_types: None, + bucket_key_enabled: None, + }], + }; + metadata.encryption_config_xml = serialize(&config).expect("sse config serializes"); + metadata.sse_config = Some(config); + set_bucket_metadata(bucket.to_string(), metadata) + .await + .expect("install bucket default SSE"); + } + + /// backlog#2368 B1: an SSE-C request suppresses the bucket default, the way + /// COPY already did. PUT passed a hard-coded `has_explicit_ssec = false`, + /// so the default filled in a managed algorithm and the request then failed + /// its own mutual-exclusion check — every bucket with default encryption + /// refused SSE-C outright. + #[tokio::test] + #[serial_test::serial] + async fn sse_c_put_is_accepted_on_a_bucket_with_default_encryption() { + use md5::{Digest as _, Md5}; + + for (algorithm, kms_key_id, prefix) in [ + (ServerSideEncryption::AES256, None, "ssec-over-aes256-default"), + (ServerSideEncryption::AWS_KMS, Some("bucket-key"), "ssec-over-kms-default"), + ] { + let (store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket(prefix, 1 << 20).await; + install_bucket_default_sse_for_test(&bucket, algorithm, kms_key_id).await; + + let customer_key = [0x2a_u8; 32]; + let key_md5 = { + let mut hasher = Md5::new(); + hasher.update(customer_key); + base64_simd::STANDARD.encode_to_string(hasher.finalize()) + }; + let payload = Bytes::from_static(b"customer-key protected payload"); + let input = PutObjectInput::builder() + .bucket(bucket.clone()) + .key("ledger.csv".to_string()) + .body(Some(StreamingBlob::from(s3s::Body::from(payload.clone())))) + .content_length(Some(i64::try_from(payload.len()).expect("test payload length must fit i64"))) + .sse_customer_algorithm(Some("AES256".to_string())) + .sse_customer_key(Some(base64_simd::STANDARD.encode_to_string(customer_key))) + .sse_customer_key_md5(Some(key_md5)) + .build() + .expect("SSE-C PUT input must build"); + + DefaultObjectUsecase::from_global() + .execute_put_object(&FS::new(), build_request(input, Method::PUT)) + .await + .unwrap_or_else(|err| panic!("a {algorithm} default bucket must accept an SSE-C PUT: {err:?}")); + + let stored = store + .get_object_info(&bucket, "ledger.csv", &ObjectOptions::default()) + .await + .expect("the SSE-C object should be readable"); + assert!( + stored + .user_defined + .keys() + .any(|key| key.eq_ignore_ascii_case("x-amz-server-side-encryption-customer-algorithm")), + "the object must be stored as SSE-C: {:?}", + stored.user_defined + ); + assert!( + !stored + .user_defined + .keys() + .any(|key| key.eq_ignore_ascii_case("x-amz-server-side-encryption-aws-kms-key-id")), + "the bucket default must not attach a KMS key to an SSE-C object: {:?}", + stored.user_defined + ); + assert!( + !stored + .user_defined + .values() + .any(|value| value == ServerSideEncryption::AWS_KMS), + "the bucket default must not claim managed encryption on an SSE-C object: {:?}", + stored.user_defined + ); + } + } + #[tokio::test] #[serial_test::serial] async fn quota_rejects_ciphertext_replication_before_polling_the_body() { diff --git a/rustfs/src/app/object/restore.rs b/rustfs/src/app/object/restore.rs index 506826281..f941e7e3d 100644 --- a/rustfs/src/app/object/restore.rs +++ b/rustfs/src/app/object/restore.rs @@ -420,7 +420,11 @@ impl DefaultObjectUsecase { ) .await .map_err(ApiError::from)?; - rustfs_scanner::record_dirty_usage_object(&bucket, &object); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &object, + rustfs_scanner::SegmentInvalidationProducerIdentity::TierTransition, + ); #[cfg(test)] maybe_pause_after_restore_status_commit(&bucket, &object).await; drop(superseded_worker_guard.take()); @@ -494,7 +498,11 @@ impl DefaultObjectUsecase { err.to_string() ); } else { - rustfs_scanner::record_dirty_usage_object(&bucket_clone, &object_clone); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket_clone, + &object_clone, + rustfs_scanner::SegmentInvalidationProducerIdentity::TierTransition, + ); debug!(bucket = %bucket_clone, object = %object_clone, "Transitioned object restored"); } }); diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index 339e0c969..063203e07 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -16,6 +16,7 @@ use super::*; use crate::on_demand_migration::{OdmStateError, PolicyConfig, SourceErrorPolicy, SourceHead}; +use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID}; pub(super) const RUSTFS_EXPECTED_CURRENT_VERSION_ID: &str = "x-rustfs-expected-current-version-id"; @@ -31,6 +32,87 @@ pub(super) const LOG_COMPONENT_APP: &str = "app"; pub(super) const LOG_SUBSYSTEM_OBJECT: &str = "object"; +fn is_delete_marker_read_error(err: &S3Error, version_id: Option<&str>) -> bool { + let code = if version_id.is_some() { + S3ErrorCode::MethodNotAllowed + } else { + S3ErrorCode::NoSuchKey + }; + err.code() == &code && err.status_code() == code.status_code() +} + +/// A delete marker is an error response, but its identity is still part of the +/// S3 read contract. Never attach the identity of a different explicit version +/// or turn an unrelated failure into a marker response. +pub(super) fn with_delete_marker_read_headers(mut err: S3Error, info: &ObjectInfo, version_id: Option<&str>) -> S3Error { + if !info.delete_marker || !is_delete_marker_read_error(&err, version_id) { + return err; + } + let marker_version = info.version_id.unwrap_or_else(Uuid::nil); + if let Some(requested) = version_id { + let requested = if requested.eq_ignore_ascii_case(NULL_VERSION_ID) { + Ok(Uuid::nil()) + } else { + Uuid::parse_str(requested) + }; + if requested.ok() != Some(marker_version) { + return err; + } + } + let version = if marker_version.is_nil() { + HeaderValue::from_static(NULL_VERSION_ID) + } else { + let Ok(version) = HeaderValue::from_str(&marker_version.to_string()) else { + return err; + }; + version + }; + let mut headers = err.headers().cloned().unwrap_or_default(); + // s3s replaces, rather than extends, the serialized error's header map. + // Keep its XML content type when adding our custom error headers. + headers + .entry(http::header::CONTENT_TYPE) + .or_insert(HeaderValue::from_static("application/xml")); + headers.insert(X_AMZ_DELETE_MARKER, HeaderValue::from_static("true")); + headers.insert(X_AMZ_VERSION_ID, version); + if version_id.is_some() + && let Some(mod_time) = info.mod_time + && let Ok(date) = mod_time.to_offset(time::UtcOffset::UTC).format(&RFC1123) + && let Ok(value) = HeaderValue::from_str(&date) + { + headers.insert(http::header::LAST_MODIFIED, value); + } + err.set_headers(headers); + err +} + +/// Recover marker metadata only after a local read and its existing fallbacks +/// have failed. Successful reads, unversioned misses and other error classes +/// do not pay for another lookup. A racing PUT/purge or failed metadata lookup +/// must keep the original failure, never resurrect an object or invent an ID. +pub(super) async fn enrich_delete_marker_read_error( + store: &ECStore, + bucket: &str, + key: &str, + opts: &ObjectOptions, + err: S3Error, +) -> S3Error { + if !(opts.versioned || opts.version_suspended || opts.version_id.is_some()) + || !is_delete_marker_read_error(&err, opts.version_id.as_deref()) + { + return err; + } + let mut metadata_opts = opts.clone(); + // The read already chose its error. This lookup supplies identity only; + // object-body conditions cannot replace that error or hide its marker. + metadata_opts.http_preconditions = None; + metadata_opts.part_number = None; + match store.get_object_info_for_delete(bucket, key, &metadata_opts).await { + Ok(info) => with_delete_marker_read_headers(err, &info, opts.version_id.as_deref()), + Err(_) => err, + } +} + pub(super) fn decoded_content_length_from_headers(headers: &HeaderMap) -> S3Result> { let Some(val) = headers.get(AMZ_DECODED_CONTENT_LENGTH) else { return Ok(None); @@ -240,10 +322,8 @@ pub(super) fn has_put_sse_request_headers(headers: &HeaderMap) -> bool { /// A request-level value always wins; the bucket default only fills a gap, and /// the unknown-algorithm fallback lives once in [`bucket_default_write_sse`]. /// -/// `has_explicit_ssec` suppresses the default entirely. Only COPY passes `true` -/// today: its destination may carry SSE-C, which must not also be given managed -/// encryption. PUT and extract pass `false`, matching their current behaviour — -/// see backlog#1826 for the divergence that leaves. +/// `has_explicit_ssec` suppresses the default entirely: an SSE-C destination +/// must not also be given managed encryption. /// /// Callers layering further overrides (PUT's `ciphertext_passthrough`) apply /// them to the returned pair. @@ -263,7 +343,14 @@ pub(super) fn resolve_bucket_default_sse( }; let effective_sse = requested_sse.or_else(|| bucket_default().map(bucket_default_write_sse)); - let effective_kms_key_id = requested_kms_key_id.or_else(|| bucket_default().and_then(|sse| sse.kms_master_key_id.clone())); + let effective_kms_key_id = if effective_sse + .as_ref() + .is_some_and(|sse| sse.as_str() == ServerSideEncryption::AWS_KMS) + { + requested_kms_key_id.or_else(|| bucket_default().and_then(|sse| sse.kms_master_key_id.clone())) + } else { + requested_kms_key_id + }; (effective_sse, effective_kms_key_id) } @@ -1014,6 +1101,117 @@ mod tests { }; use std::sync::Arc; + #[test] + fn delete_marker_read_headers_round_trip_uuid_and_null_errors() { + let uuid = Uuid::parse_str("9341ae04-d4ce-468c-a4e1-6501d58cd6b7").unwrap(); + let modified = time::macros::datetime!(2026-09-09 12:30:45 +08:00); + for stored_version in [Some(uuid), Some(Uuid::nil()), None] { + let expected_version = stored_version + .filter(|id| !id.is_nil()) + .map(|id| id.to_string()) + .unwrap_or_else(|| "null".to_string()); + let info = ObjectInfo { + delete_marker: true, + version_id: stored_version, + mod_time: Some(modified), + ..Default::default() + }; + let explicit = stored_version.unwrap_or_else(Uuid::nil).to_string().to_uppercase(); + for requested in [None, Some(explicit.as_str()), Some(expected_version.as_str())] { + let code = if requested.is_some() { + S3ErrorCode::MethodNotAllowed + } else { + S3ErrorCode::NoSuchKey + }; + let status = code.status_code().unwrap(); + let error = with_delete_marker_read_headers(S3Error::new(code), &info, requested); + let response = error.to_http_response().expect("marker error must serialize"); + assert_eq!(response.status(), status); + assert_eq!(response.headers()[http::header::CONTENT_TYPE], "application/xml"); + assert_eq!(response.headers()[X_AMZ_DELETE_MARKER], "true"); + assert_eq!(response.headers()[X_AMZ_VERSION_ID], expected_version); + if requested.is_some() { + assert_eq!(response.headers()[http::header::LAST_MODIFIED], "Wed, 09 Sep 2026 04:30:45 GMT"); + } else { + assert!(!response.headers().contains_key(http::header::LAST_MODIFIED)); + } + } + } + } + + #[test] + fn delete_marker_read_headers_preserve_error_context() { + let info = ObjectInfo { + delete_marker: true, + version_id: Some(Uuid::new_v4()), + ..Default::default() + }; + let mut original = S3Error::with_message(S3ErrorCode::NoSuchKey, "original local read failure"); + original.set_source(Box::new(io::Error::other("original storage cause"))); + original.set_request_id("request-id"); + original.set_status_code(StatusCode::NOT_FOUND); + let mut headers = HeaderMap::new(); + headers.insert("x-test-existing", HeaderValue::from_static("preserved")); + headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/custom+xml")); + original.set_headers(headers); + let error = with_delete_marker_read_headers(original, &info, None); + assert_eq!(error.code(), &S3ErrorCode::NoSuchKey); + assert_eq!(error.status_code(), Some(StatusCode::NOT_FOUND)); + assert_eq!(error.message(), Some("original local read failure")); + assert_eq!(error.request_id(), Some("request-id")); + assert_eq!(error.source().unwrap().to_string(), "original storage cause"); + assert_eq!(error.headers().unwrap()["x-test-existing"], "preserved"); + assert_eq!(error.headers().unwrap()[http::header::CONTENT_TYPE], "application/custom+xml"); + assert_eq!(error.headers().unwrap()[X_AMZ_DELETE_MARKER], "true"); + + let explicit = info.version_id.unwrap().to_string(); + let error = with_delete_marker_read_headers(S3Error::new(S3ErrorCode::MethodNotAllowed), &info, Some(&explicit)); + assert!( + !error.headers().unwrap().contains_key(http::header::LAST_MODIFIED), + "missing metadata cannot invent a date" + ); + } + + #[test] + fn delete_marker_read_headers_reject_unrelated_failures_and_versions() { + let marker_id = Uuid::new_v4(); + let version = marker_id.to_string(); + let mut info = ObjectInfo { + delete_marker: true, + version_id: Some(marker_id), + ..Default::default() + }; + for (requested, code) in [ + (None, S3ErrorCode::AccessDenied), + (None, S3ErrorCode::InternalError), + (None, S3ErrorCode::PreconditionFailed), + (None, S3ErrorCode::NotModified), + (None, S3ErrorCode::NoSuchVersion), + (None, S3ErrorCode::MethodNotAllowed), + (Some(version.as_str()), S3ErrorCode::NoSuchKey), + (Some(version.as_str()), S3ErrorCode::AccessDenied), + ] { + let error = with_delete_marker_read_headers(S3Error::new(code), &info, requested); + assert!(error.headers().is_none()); + } + let wrong_version = Uuid::new_v4().to_string(); + for requested in [wrong_version.as_str(), "null", "", "not-a-version", "bad\r\nheader: injected"] { + let error = with_delete_marker_read_headers(S3Error::new(S3ErrorCode::MethodNotAllowed), &info, Some(requested)); + assert!(error.headers().is_none()); + } + let mut original = S3Error::new(S3ErrorCode::NoSuchKey); + original.set_status_code(StatusCode::FORBIDDEN); + let error = with_delete_marker_read_headers(original, &info, None); + assert_eq!(error.status_code(), Some(StatusCode::FORBIDDEN)); + assert!(error.headers().is_none()); + + info.delete_marker = false; + let error = with_delete_marker_read_headers(S3Error::new(S3ErrorCode::NoSuchKey), &info, None); + assert!(error.headers().is_none(), "a racing PUT is not a delete marker"); + let error = with_delete_marker_read_headers(S3Error::new(S3ErrorCode::MethodNotAllowed), &info, Some(&version)); + assert!(error.headers().is_none()); + } + #[test] fn parse_expires_header_accepts_http_date() { let expires = parse_expires_header(Some("Wed, 21 Oct 2015 07:28:00 GMT")) @@ -1187,6 +1385,21 @@ mod tests { assert_eq!(kms_key_id.as_deref(), Some("request-key")); } + #[test] + fn resolve_bucket_default_sse_does_not_inherit_a_kms_key_for_an_explicit_sse_s3_request() { + let config = bucket_sse_config_with(ServerSideEncryption::AWS_KMS, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse( + Some(&config), + Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), + None, + false, + ); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AES256)); + assert!(kms_key_id.is_none(), "an SSE-S3 request must not inherit the bucket KMS key"); + } + #[test] fn resolve_bucket_default_sse_fills_gaps_from_the_bucket_default() { let config = bucket_sse_config_with(ServerSideEncryption::AWS_KMS, Some("bucket-key")); diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index 48ea1ce50..9e67b90ec 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -689,8 +689,8 @@ fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()> )); } validate_single_byte(csv.comments.as_deref(), S3ErrorCode::InvalidRequestParameter)?; - validate_single_byte(csv.quote_character.as_deref(), S3ErrorCode::InvalidRequestParameter)?; - validate_single_byte(csv.quote_escape_character.as_deref(), S3ErrorCode::InvalidRequestParameter)?; + validate_single_character(csv.quote_character.as_deref())?; + validate_single_character(csv.quote_escape_character.as_deref())?; validate_input_record_delimiter(csv.record_delimiter.as_deref())?; validate_input_delimiter_pair(csv.field_delimiter.as_deref(), csv.record_delimiter.as_deref())?; } @@ -778,6 +778,15 @@ fn invalid_scan_range_error() -> S3Error { S3Error::with_message(S3ErrorCode::InvalidRequestParameter, INVALID_SCAN_RANGE_MESSAGE.to_string()) } +fn validate_single_character(value: Option<&str>) -> S3Result<()> { + if let Some(value) = value + && value.chars().count() != 1 + { + return Err(S3Error::new(S3ErrorCode::InvalidRequestParameter)); + } + Ok(()) +} + fn validate_single_byte(value: Option<&str>, code: S3ErrorCode) -> S3Result<()> { if let Some(value) = value && value.len() != 1 @@ -3524,6 +3533,29 @@ mod tests { assert_eq!(error.message(), Some(INVALID_SCAN_RANGE_MESSAGE)); } + #[test] + fn validate_accepts_single_unicode_csv_input_quotes() { + for quote in ["ع", "界", "🦀"] { + let mut input = base_input(); + let csv = input.request.input_serialization.csv.as_mut().expect("CSV input"); + csv.quote_character = Some(quote.to_owned()); + csv.quote_escape_character = Some(quote.to_owned()); + validate_select_request(&HeaderMap::new(), &mut input).expect("one Unicode scalar is a valid CSV quote"); + } + for quote in ["", "عع", "e\u{301}"] { + let mut input = base_input(); + input + .request + .input_serialization + .csv + .as_mut() + .expect("CSV input") + .quote_character = Some(quote.to_owned()); + let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("quote must be one scalar"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequestParameter); + } + } + #[test] fn validate_rejects_unknown_csv_header_mode_before_streaming() { let mut input = base_input(); diff --git a/rustfs/src/cluster_snapshot.rs b/rustfs/src/cluster_snapshot.rs index 9d38302a8..fa72d8fbd 100644 --- a/rustfs/src/cluster_snapshot.rs +++ b/rustfs/src/cluster_snapshot.rs @@ -277,6 +277,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, }); assert_eq!(ready.state, ClusterRuntimeReadinessState::Ready); @@ -288,6 +289,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![ReadinessDegradedReason::StorageQuorumUnavailable], + storage_details: None, }); assert_eq!(degraded.state, ClusterRuntimeReadinessState::Degraded); assert_eq!(degraded.degraded_reasons, vec![ReadinessDegradedReason::StorageQuorumUnavailable]); @@ -304,6 +306,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: vec![ReadinessDegradedReason::StorageAndLockUnavailable], + storage_details: None, }); let snapshot = cluster_read_only_snapshot_from_endpoint_pools(&endpoint_pools, runtime_status); diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index c6a9f9b19..91439e458 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -24,6 +24,77 @@ const MAX_VERSIONS_EXCEEDED_MESSAGE: &str = "You've exceeded the limit on the nu /// S3 error code for a request that names a KMS key the KMS does not hold. pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException"; +/// Map a KMS failure that surfaced on the S3 data path to its S3 error code. +/// +/// The contract deliberately differs from the admin lifecycle handlers +/// (`kms_key_lifecycle::lifecycle_error_status`): there a key id is the +/// resource being addressed, so a missing key is `404`. Here the key id +/// arrives inside a request header or a bucket default, so a key that is +/// missing, disabled, or otherwise unusable is configuration the caller has to +/// correct — AWS answers `400`, and reporting `500` instead both misfiles the +/// failure as a server fault and makes SDKs back off and retry a request that +/// cannot succeed. +/// +/// `None` keeps the caller's fallthrough, which is the `500` that integrity +/// faults — damaged, unreadable, or unknown-format key material — must keep. +/// +/// Messages either name what the caller asked for or stay generic; detail that +/// belongs to the deployment rather than the request stays in `source`, which +/// the caller attaches and the logs retain. +fn data_plane_kms_error(error: &rustfs_kms::KmsError) -> Option<(S3ErrorCode, String)> { + use rustfs_kms::KmsError as Kms; + + let generic = |code: S3ErrorCode| { + let message = ApiError::error_code_to_message(&code); + Some((code, message)) + }; + + match error { + Kms::KeyNotFound { key_id } => Some(( + S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()), + format!("KMS key not found: {key_id}"), + )), + Kms::KeyVersionNotFound { key_id, version } => Some(( + S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()), + format!("KMS key version {version} not found for key {key_id}"), + )), + // The key exists but its state forbids the operation (disabled, + // pending deletion). AWS treats an invalid key state as a request + // error, not a server fault. + Kms::InvalidOperation { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())), + // A policy decision needs a human, so it must be distinguishable from + // the transient classes an SDK retries. + Kms::AccessDenied { .. } => generic(S3ErrorCode::AccessDenied), + // Request-side faults: what was asked for cannot be served as asked. + Kms::ContextMismatch { .. } + | Kms::InvalidKey { .. } + | Kms::ValidationError { .. } + | Kms::UnsupportedAlgorithm { .. } + | Kms::InvalidKeySize { .. } => Some((S3ErrorCode::InvalidRequest, error.to_string())), + // A deployment whose KMS configuration cannot serve the request (for + // example no key named and no default). Actionable, but the detail + // describes the deployment, so it stays out of the response body. + Kms::ConfigurationError { .. } => { + Some((S3ErrorCode::InvalidRequest, "The KMS configuration cannot serve this request".to_string())) + } + // Transient: worth retrying, and must be counted against availability + // rather than against the caller. `IoError` belongs here because it is + // how a backend reports that its key store itself was unreachable — + // rustfs/rustfs#7470 separated that from a missing key precisely so the + // two stop looking alike, and leaving it on the 500 fallthrough would + // erase that distinction again at the S3 boundary. + Kms::BackendError { .. } + | Kms::IoError { .. } + | Kms::OperationTimedOut { .. } + | Kms::OperationCancelled { .. } + | Kms::CredentialsUnavailable { .. } + | Kms::CacheError { .. } => generic(S3ErrorCode::ServiceUnavailable), + // A permanent gap in the configured backend, never a missing resource. + Kms::UnsupportedCapability { .. } => Some((S3ErrorCode::NotImplemented, error.to_string())), + _ => None, + } +} + /// HTTP status of the error codes s3s cannot derive on its own. /// /// s3s answers `None` for every `Custom` code, which the response layer turns @@ -472,12 +543,7 @@ impl From for ApiError { }; } - if inner.downcast_ref::().is_some() - || matches!( - inner.downcast_ref::(), - Some(rustfs_kms::KmsError::BackendError { .. }) - ) - { + if inner.downcast_ref::().is_some() { return ApiError { code: S3ErrorCode::ServiceUnavailable, message: ApiError::error_code_to_message(&S3ErrorCode::ServiceUnavailable), @@ -485,14 +551,11 @@ impl From for ApiError { }; } - // A request header or bucket default naming a key the KMS does not - // hold is the caller's mistake to correct, and S3 reports it as - // 400 `KMS.NotFoundException`. Left to the fallthrough it became a - // 500 whose generic message hid which key was missing. - if let Some(rustfs_kms::KmsError::KeyNotFound { key_id }) = inner.downcast_ref::() { - let message = format!("KMS key not found: {key_id}"); + if let Some(kms_error) = inner.downcast_ref::() + && let Some((code, message)) = data_plane_kms_error(kms_error) + { return ApiError { - code: S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()), + code, message, source: Some(Box::new(err)), }; @@ -1060,6 +1123,91 @@ mod tests { assert_eq!(s3_error.status_code(), Some(StatusCode::BAD_REQUEST)); } + /// backlog#2368 B6: every KMS failure class that is not an integrity fault + /// carries a status that says whether retrying, fixing the request, or + /// calling a human is the right response. Collapsing them onto 500 made + /// SDKs back off on unfixable configuration errors and filed every one of + /// them as a server fault. + #[test] + fn kms_data_plane_errors_are_classified_by_what_the_caller_should_do() { + let cases: Vec<(rustfs_kms::KmsError, S3ErrorCode)> = vec![ + // Names something the KMS does not hold. + ( + rustfs_kms::KmsError::key_version_not_found("finance-key", 7), + S3ErrorCode::Custom(KMS_KEY_NOT_FOUND_ERROR_CODE.into()), + ), + // The key exists but its state forbids the operation. + (rustfs_kms::KmsError::invalid_key_state("disabled"), S3ErrorCode::InvalidRequest), + // Request-side faults. + (rustfs_kms::KmsError::context_mismatch("bucket differs"), S3ErrorCode::InvalidRequest), + (rustfs_kms::KmsError::invalid_key("malformed key id"), S3ErrorCode::InvalidRequest), + (rustfs_kms::KmsError::validation_error("empty key id"), S3ErrorCode::InvalidRequest), + (rustfs_kms::KmsError::unsupported_algorithm("aes-999"), S3ErrorCode::InvalidRequest), + (rustfs_kms::KmsError::invalid_key_size(32, 16), S3ErrorCode::InvalidRequest), + ( + rustfs_kms::KmsError::configuration_error("no default key configured"), + S3ErrorCode::InvalidRequest, + ), + // A policy decision that needs a human, never retried by an SDK. + (rustfs_kms::KmsError::access_denied("no kms:Decrypt grant"), S3ErrorCode::AccessDenied), + // Transient, worth retrying, counted against availability. + (rustfs_kms::KmsError::backend_error("vault refused"), S3ErrorCode::ServiceUnavailable), + ( + rustfs_kms::KmsError::operation_timed_out("attempt deadline"), + S3ErrorCode::ServiceUnavailable, + ), + ( + rustfs_kms::KmsError::operation_cancelled("shutting down"), + S3ErrorCode::ServiceUnavailable, + ), + ( + rustfs_kms::KmsError::credentials_unavailable("approle login failed"), + S3ErrorCode::ServiceUnavailable, + ), + (rustfs_kms::KmsError::cache_error("poisoned"), S3ErrorCode::ServiceUnavailable), + // A key store that cannot be read is an outage, not a missing key: + // rustfs/rustfs#7470 made the backend say so, and the S3 boundary + // has to keep the two apart. + ( + rustfs_kms::KmsError::io_error("No such file or directory (os error 2)"), + S3ErrorCode::ServiceUnavailable, + ), + // A permanent gap in the configured backend, never a missing resource. + ( + rustfs_kms::KmsError::unsupported_capability("local", "rewrap"), + S3ErrorCode::NotImplemented, + ), + ]; + + for (error, expected) in cases { + let description = error.to_string(); + let api_error = ApiError::from(StorageError::other(error)); + assert_eq!(api_error.code, expected, "wrong code for: {description}"); + assert_ne!(api_error.code, S3ErrorCode::InternalError, "must not be a server fault: {description}"); + } + } + + /// A deployment-side configuration message describes the server, not the + /// request, so it stays in `source` the way the storage-IO mapping does. + #[test] + fn kms_configuration_errors_do_not_echo_deployment_detail() { + let detail = "vault mount /secret/rustfs-prod has no default key"; + let api_error = ApiError::from(StorageError::other(rustfs_kms::KmsError::configuration_error(detail))); + + assert_eq!(api_error.code, S3ErrorCode::InvalidRequest); + assert!( + !api_error.message.contains(detail), + "message leaked deployment detail: {}", + api_error.message + ); + let source = api_error + .source + .as_deref() + .and_then(|source| source.downcast_ref::()) + .expect("API error should retain the storage error source"); + assert!(source.to_string().contains(detail), "the detail must survive on the source"); + } + #[test] fn test_generated_error_codes_keep_their_own_status() { let s3_error = S3Error::from(ApiError::from(StorageError::other(rustfs_kms::KmsError::backend_error("down")))); diff --git a/rustfs/src/server/health.rs b/rustfs/src/server/health.rs index 47f4357c1..ee1747263 100644 --- a/rustfs/src/server/health.rs +++ b/rustfs/src/server/health.rs @@ -343,7 +343,7 @@ pub(crate) fn build_health_response_parts( let payload = if method == Method::HEAD { None } else { - Some(build_health_payload(HealthPayloadContext { + let mut payload = build_health_payload(HealthPayloadContext { probe, health, storage_ready, @@ -354,7 +354,18 @@ pub(crate) fn build_health_response_parts( uptime, kms_ready, include_dependency_details, - })) + }); + if let Some(details) = readiness_report.and_then(|report| report.storage_details) + && payload.get("details").is_some() + { + payload["details"]["storage"]["readQuorum"] = json!(details.read_quorum_ready); + payload["details"]["storage"]["writeQuorum"] = json!(details.write_quorum_ready); + payload["details"]["poolMetadata"] = json!({ + "ready": details.pool_metadata_write_ready, + "status": if details.pool_metadata_write_ready { "writable" } else { "unavailable" }, + }); + } + Some(payload) }; HealthResponseParts { @@ -390,6 +401,14 @@ pub(crate) fn build_health_payload(ctx: HealthPayloadContext<'_>) -> Value { if ctx.include_dependency_details { payload["details"] = build_component_details(ctx.storage_ready, ctx.iam_ready, ctx.lock_quorum_ready, ctx.kms_ready); + payload["details"]["storage"]["readinessScope"] = json!(match ctx.probe { + HealthProbe::ClusterRead => "read_quorum", + _ => "write_quorum_and_pool_metadata", + }); + payload["details"]["storage"]["source"] = json!(match ctx.probe { + HealthProbe::Readiness => "local_runtime", + _ => "storage_inventory", + }); payload["degradedReasons"] = build_degraded_reasons(ctx.degraded_reasons); } @@ -432,9 +451,86 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, } } + #[test] + #[serial] + fn node_storage_details_separate_quorum_from_pool_metadata() { + with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || { + for (read_quorum, write_quorum, metadata_ready, lock_ready) in [ + (true, true, true, true), + (true, false, true, false), + (false, false, true, false), + (true, true, false, true), + (true, true, true, true), + ] { + let mut report = ready_report(); + report.readiness.storage_ready = write_quorum && metadata_ready; + report.readiness.lock_quorum_ready = lock_ready; + report.storage_details = Some(crate::shared_types::StorageReadinessDetails { + read_quorum_ready: read_quorum, + write_quorum_ready: write_quorum, + pool_metadata_write_ready: metadata_ready, + }); + let parts = build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs", None, None); + let expected_ready = write_quorum && metadata_ready && lock_ready; + assert_eq!( + parts.status_code, + if expected_ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } + ); + let payload = parts.payload.expect("GET readiness body"); + assert_eq!(payload["ready"], expected_ready); + assert_eq!(payload["details"]["storage"]["ready"], write_quorum && metadata_ready); + assert_eq!( + payload["details"]["storage"]["status"], + if write_quorum && metadata_ready { + "connected" + } else { + "disconnected" + } + ); + assert_eq!(payload["details"]["storage"]["readQuorum"], read_quorum); + assert_eq!(payload["details"]["storage"]["writeQuorum"], write_quorum); + assert_eq!(payload["details"]["poolMetadata"]["ready"], metadata_ready); + assert_eq!(payload["details"]["storage"]["source"], "local_runtime"); + assert_eq!(payload["details"]["storage"]["readinessScope"], "write_quorum_and_pool_metadata"); + assert!( + build_health_response_parts(Method::HEAD, HealthProbe::Readiness, Some(&report), "rustfs", None, None) + .payload + .is_none() + ); + } + }); + } + + #[test] + #[serial] + fn node_storage_details_do_not_expand_minimal_or_liveness_payloads() { + let mut report = ready_report(); + report.storage_details = Some(crate::shared_types::StorageReadinessDetails { + read_quorum_ready: true, + write_quorum_ready: true, + pool_metadata_write_ready: true, + }); + with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("true"), || { + let parts = build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs", None, None); + assert_eq!(parts.payload, Some(json!({ "status": "ok", "ready": true }))); + }); + with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || { + let parts = build_health_response_parts(Method::GET, HealthProbe::Liveness, Some(&report), "rustfs", None, None); + assert_eq!(parts.status_code, StatusCode::OK); + let payload = parts.payload.expect("liveness GET body"); + assert!(payload.get("details").is_none()); + assert!(payload.get("ready").is_none()); + }); + } + #[tokio::test] async fn readiness_collects_object_stalls_and_recovers_on_completion() { let object_traffic_health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO); diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 4b035b8da..36583050a 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -28,6 +28,7 @@ use crate::server::{ StsQueryApiCompatLayer, VirtualHostStyleHintLayer, redact_sensitive_uri_query, }, rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env}, + ssec_transport::SsecTransportLayer, strip_valid_port_suffix, tls_material::{ TlsAcceptFailure, TlsAcceptorHolder, TlsHandshakeFailureKind, accept_tls_with_deadline, build_acceptor_from_loaded, @@ -1848,6 +1849,11 @@ fn process_connection( request_body_idle_timeout, } = context; + // Whether this listener terminated TLS for this connection; the SSE-C + // transport policy needs the connection's own answer, not a + // deployment-wide setting. + let connection_is_tls = tls_acceptor.is_some(); + // Build the hybrid service per-connection. // Note: NodeService is not Clone (holds LocalPeerS3Client), and the SwiftService // type is feature-gated, so we cannot pre-build the full hybrid service. @@ -1966,6 +1972,14 @@ fn process_connection( // a spoof-proof client IP. Absent (None) unless enabled via // RUSTFS_API_RATE_LIMIT_ENABLE with a non-zero RPM. .option_layer(rate_limit_layer.clone()) + // backlog#2369 P7.2: an SSE-C request carries the customer key + // in a header, so a plaintext hop leaks it permanently. Sits + // beside the rate limiter: after the trusted-proxy layer, which + // is what makes a forwarded `https` protocol trustworthy, and + // after the request context so a rejection can echo the request + // id. Reports by default; refuses only under + // RUSTFS_SSE_C_REQUIRE_TLS. + .layer(SsecTransportLayer::new(connection_is_tls)) // CRITICAL: Insert ReadinessGateLayer before business logic // This stops requests from hitting IAMAuth or Storage if they are not ready. .layer(ReadinessGateLayer::new(readiness.clone())) diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index 5e245b9f1..01e5dadb9 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -27,6 +27,7 @@ mod readiness; mod runtime; pub(crate) mod runtime_sources; mod service_state; +mod ssec_transport; pub mod tls_material; use tracing::warn; diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index f14b77390..947b77e39 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -17,8 +17,8 @@ use crate::server::{ServiceState, ServiceStateManager}; use crate::server::{has_path_prefix, is_table_catalog_path}; use crate::storage_api::cluster::control_plane::ClusterControlPlane; use crate::storage_api::error::StorageError; -use crate::storage_api::server::readiness::contract::admin::StorageAdminApi; -use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure}; +use crate::storage_api::server::readiness::contract::admin::{DiskSetSelector, StorageAdminApi}; +use crate::storage_api::server::readiness::{DiskStore, Endpoint, EndpointServerPools, disk_endpoint_snapshot, is_dist_erasure}; #[cfg(test)] use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints}; use crate::storage_api::startup::shutdown::mark_get_metadata_read_version_coalescing_service_ready; @@ -30,7 +30,7 @@ use http_body_util::{BodyExt, Full}; use hyper::body::Incoming; use metrics::{counter, gauge}; use rustfs_common::GlobalReadiness; -use rustfs_madmin::{Disk, StorageInfo}; +use rustfs_madmin::{BackendInfo, Disk, StorageInfo}; use std::future::Future; use std::pin::Pin; use std::sync::Arc; @@ -76,7 +76,7 @@ fn startup_runtime_readiness_max_wait() -> Duration { const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready"; const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total"; -pub use crate::shared_types::{DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason}; +pub use crate::shared_types::{DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason, StorageReadinessDetails}; /// ReadinessGateLayer ensures that the system components (IAM, Storage) /// are fully initialized before allowing any request to proceed. @@ -288,10 +288,16 @@ fn pool_metadata_write_readiness(result: Result<(), StorageError>) -> StorageWri } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct LockQuorumCacheEntry { captured_at: Instant, + observation: LockQuorumObservation, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct LockQuorumObservation { status: LockQuorumStatus, + online_hosts: HashSet, } #[derive(Debug, Clone)] @@ -317,6 +323,7 @@ pub struct LockQuorumStatus { const DISK_STATE_OK: &str = "ok"; const DISK_STATE_UNFORMATTED: &str = "unformatted"; const RUNTIME_STATE_RETURNING: &str = "returning"; +const NODE_STORAGE_READINESS_TIMEOUT: Duration = Duration::from_millis(100); fn health_readiness_cache_ttl() -> Duration { Duration::from_millis(rustfs_utils::get_env_u64( @@ -439,7 +446,7 @@ async fn update_storage_readiness_cache(status: StorageWriteReadinessStatus) { }); } -async fn load_cached_lock_quorum_status() -> Option { +async fn load_cached_lock_quorum_status() -> Option { let ttl = health_readiness_cache_ttl(); if ttl.is_zero() { return None; @@ -448,13 +455,13 @@ async fn load_cached_lock_quorum_status() -> Option { let cache = lock_quorum_status_cache().lock().await; let entry = cache.as_ref()?; if entry.captured_at.elapsed() <= ttl { - return Some(entry.status); + return Some(entry.observation.clone()); } None } -async fn update_lock_quorum_status_cache(status: LockQuorumStatus) { +async fn update_lock_quorum_status_cache(observation: LockQuorumObservation) { if health_readiness_cache_ttl().is_zero() { return; } @@ -462,7 +469,7 @@ async fn update_lock_quorum_status_cache(status: LockQuorumStatus) { let mut cache = lock_quorum_status_cache().lock().await; *cache = Some(LockQuorumCacheEntry { captured_at: Instant::now(), - status, + observation, }); } @@ -730,6 +737,7 @@ fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) -> DependencyReadinessReport { degraded_reasons: degraded_reasons(readiness), readiness, + storage_details: None, } } @@ -740,6 +748,7 @@ fn dependency_readiness_report_from_write_status( DependencyReadinessReport { degraded_reasons: degraded_reasons_with_pool_meta_status(readiness, storage.pool_metadata_reason), readiness, + storage_details: None, } } @@ -774,18 +783,95 @@ pub async fn collect_cluster_read_health_report() -> DependencyReadinessReport { } pub async fn collect_node_readiness_report() -> DependencyReadinessReport { - let storage = node_pool_meta_write_readiness().await; + let lock_observation = collect_lock_quorum_observation().await; + let mut storage = StorageWriteReadinessStatus::default(); + let mut details = StorageReadinessDetails::default(); + let mut storage_check_timed_out = false; + if let Some(store) = runtime_sources::current_object_store_handle() { + storage = pool_metadata_write_readiness(store.pool_meta_write_status().await); + details.pool_metadata_write_ready = storage.ready; + match node_storage_snapshot(store.as_ref(), &lock_observation.online_hosts).await { + Ok(info) => { + details.read_quorum_ready = storage_read_ready_from_runtime_state(&info); + details.write_quorum_ready = storage_ready_from_runtime_state(&info); + } + Err(StorageError::Timeout) => storage_check_timed_out = true, + Err(_) => {} + } + } + storage.ready &= details.write_quorum_ready; let readiness = DependencyReadiness { storage_ready: storage.ready, iam_ready: runtime_sources::current_iam_ready(), - lock_quorum_ready: collect_lock_quorum_status().await.ready, + lock_quorum_ready: lock_observation.status.ready, peer_health_ready: collect_peer_health_readiness(), }; - let report = dependency_readiness_report_from_write_status(readiness, storage); + let mut report = dependency_readiness_report_from_write_status(readiness, storage); + report.storage_details = Some(details); + if storage_check_timed_out { + report + .degraded_reasons + .push(ReadinessDegradedReason::StorageReadinessCheckTimeout); + } record_readiness_report(&report); report } +async fn node_storage_snapshot(store: &S, online_hosts: &HashSet) -> Result +where + S: StorageAdminApi, +{ + tokio::time::timeout(NODE_STORAGE_READINESS_TIMEOUT, async { + let mut info = StorageInfo { + backend: store.backend_info().await, + ..Default::default() + }; + if configured_readiness_topology(&info).is_none() { + return Ok(info); + } + + // Inventory and runtime health are local snapshots. Reuse the lock probe's + // peer reachability so an idle remote disk cannot outlive its failed host. + // Do not perform disk-info RPCs or filesystem probes in node readiness. + for (pool_idx, &set_count) in info.backend.total_sets.iter().enumerate() { + for set_idx in 0..set_count { + for disk in store + .disk_set_inventory(DiskSetSelector::new(pool_idx, set_idx)) + .await? + .into_iter() + .flatten() + { + info.disks.push(node_disk_snapshot( + disk_endpoint_snapshot(&disk), + disk.runtime_state().as_str(), + online_hosts, + )); + } + } + } + Ok(info) + }) + .await + .map_err(|_| StorageError::Timeout)? +} + +fn node_disk_snapshot(endpoint: Endpoint, runtime_state: &str, online_hosts: &HashSet) -> Disk { + let reachable = endpoint.is_local || online_hosts.contains(&endpoint.host_port()); + // Returning drives can still reject data I/O as faulty. Without a fresh + // disk-info probe, only an Online runtime observation can supply quorum. + let online = reachable && runtime_state == rustfs_madmin::ITEM_ONLINE; + Disk { + endpoint: endpoint.to_string(), + drive_path: endpoint.get_file_path(), + pool_index: endpoint.pool_idx, + set_index: endpoint.set_idx, + disk_index: endpoint.disk_idx, + state: if online { DISK_STATE_OK } else { "offline" }.to_string(), + runtime_state: Some(runtime_state.to_string()), + ..Default::default() + } +} + async fn collect_cluster_health_report_with( kind: ClusterHealthProbeKind, mut load_report: LoadFn, @@ -820,11 +906,22 @@ fn cluster_health_timeout_report() -> DependencyReadinessReport { peer_health_ready: false, }, degraded_reasons: vec![ReadinessDegradedReason::ClusterHealthTimeout], + storage_details: None, } } async fn collect_node_readiness() -> DependencyReadiness { - collect_node_readiness_report().await.readiness + // Startup publication retains its local dependency gate. Runtime probes + // additionally report storage quorum without changing S3 admission. + let storage = node_pool_meta_write_readiness().await; + let readiness = DependencyReadiness { + storage_ready: storage.ready, + iam_ready: runtime_sources::current_iam_ready(), + lock_quorum_ready: collect_lock_quorum_status().await.ready, + peer_health_ready: collect_peer_health_readiness(), + }; + record_readiness_report(&dependency_readiness_report_from_write_status(readiness, storage)); + readiness } pub async fn collect_cluster_read_dependency_readiness_report() -> DependencyReadinessReport { @@ -856,11 +953,15 @@ pub(crate) async fn snapshot_dependency_readiness_report() -> DependencyReadines } async fn collect_lock_quorum_status() -> LockQuorumStatus { + collect_lock_quorum_observation().await.status +} + +async fn collect_lock_quorum_observation() -> LockQuorumObservation { if let Some(cached) = load_cached_lock_quorum_status().await { cached } else { - let computed = collect_lock_quorum_status_uncached().await; - update_lock_quorum_status_cache(computed).await; + let computed = collect_lock_quorum_observation_uncached().await; + update_lock_quorum_status_cache(computed.clone()).await; computed } } @@ -972,20 +1073,27 @@ fn aggregate_lock_quorum_status(pool_endpoints: &EndpointServerPools, online_hos } async fn collect_lock_quorum_status_uncached() -> LockQuorumStatus { + collect_lock_quorum_observation_uncached().await.status +} + +async fn collect_lock_quorum_observation_uncached() -> LockQuorumObservation { if !is_dist_erasure().await { - return LockQuorumStatus { - ready: true, - connected_clients: 1, - total_clients: 1, - required_quorum: 1, + return LockQuorumObservation { + status: LockQuorumStatus { + ready: true, + connected_clients: 1, + total_clients: 1, + required_quorum: 1, + }, + online_hosts: HashSet::new(), }; } let Some(pool_endpoints) = runtime_sources::current_endpoints_handle() else { - return LockQuorumStatus::default(); + return LockQuorumObservation::default(); }; let Some(lock_clients) = runtime_sources::current_lock_clients_handle() else { - return LockQuorumStatus::default(); + return LockQuorumObservation::default(); }; let online_hosts = futures::future::join_all(lock_clients.iter().map(|(host, client)| { @@ -998,7 +1106,10 @@ async fn collect_lock_quorum_status_uncached() -> LockQuorumStatus { .filter_map(|(host, online)| online.then_some(host)) .collect::>(); - aggregate_lock_quorum_status(&pool_endpoints, &online_hosts) + LockQuorumObservation { + status: aggregate_lock_quorum_status(&pool_endpoints, &online_hosts), + online_hosts, + } } pub async fn wait_for_runtime_readiness_with( @@ -1060,12 +1171,206 @@ mod tests { } use super::*; + use crate::storage_api::server::readiness::{DiskOption, new_disk}; use rustfs_madmin::{BackendInfo, Disk}; use serial_test::serial; use std::future; use std::sync::atomic::{AtomicUsize, Ordering}; use temp_env::{async_with_vars, with_var}; + #[derive(Debug, Default)] + struct RuntimeInventory { + backend: BackendInfo, + disks: HashMap>>, + pending_set: Option, + failed_set: Option, + } + + #[async_trait::async_trait] + impl StorageAdminApi for RuntimeInventory { + type BackendInfo = BackendInfo; + type StorageInfo = StorageInfo; + type Disk = DiskStore; + type Error = StorageError; + + async fn backend_info(&self) -> BackendInfo { + self.backend.clone() + } + + async fn storage_info(&self) -> StorageInfo { + panic!("node readiness must not perform a cluster storage-info probe") + } + + async fn local_storage_info(&self) -> StorageInfo { + panic!("node readiness must not perform local disk-info I/O") + } + + async fn disk_set_inventory(&self, selector: DiskSetSelector) -> Result>, StorageError> { + if self.pending_set == Some(selector) { + return future::pending().await; + } + if self.failed_set == Some(selector) { + return Err(StorageError::other("inventory unavailable")); + } + Ok(self.disks.get(&selector).cloned().unwrap_or_default()) + } + + fn set_drive_counts(&self) -> Vec { + self.backend.drives_per_set.clone() + } + } + + async fn runtime_inventory(layouts: &[(usize, usize, usize)]) -> RuntimeInventory { + let mut store = RuntimeInventory::default(); + for (pool_idx, &(set_count, drive_count, parity)) in layouts.iter().enumerate() { + store.backend.total_sets.push(set_count); + store.backend.drives_per_set.push(drive_count); + store.backend.standard_sc_data.push(drive_count - parity); + store.backend.standard_sc_parities.push(parity); + for set_idx in 0..set_count { + let mut disks = Vec::new(); + for disk_idx in 0..drive_count { + let endpoint = Endpoint { + url: url::Url::parse(&format!("http://node-{disk_idx}:9000/pool-{pool_idx}-set-{set_idx}")) + .expect("valid test endpoint"), + is_local: false, + pool_idx: i32::try_from(pool_idx).expect("test pool index"), + set_idx: i32::try_from(set_idx).expect("test set index"), + disk_idx: i32::try_from(disk_idx).expect("test disk index"), + }; + disks.push(Some( + new_disk(&endpoint, &DiskOption::default()) + .await + .expect("create runtime disk handle"), + )); + } + store.disks.insert(DiskSetSelector::new(pool_idx, set_idx), disks); + } + } + store + } + + #[tokio::test] + async fn node_storage_snapshot_tracks_read_write_quorum_and_recovery() { + for (drive_count, parity) in [(4, 2), (4, 1), (8, 2)] { + let store = runtime_inventory(&[(1, drive_count, parity)]).await; + let data = drive_count - parity; + let write_quorum = data + usize::from(data == parity); + for survivors in (0..=drive_count).rev().chain(std::iter::once(drive_count)) { + let online_hosts = (0..survivors).map(|idx| format!("node-{idx}:9000")).collect(); + let info = node_storage_snapshot(&store, &online_hosts) + .await + .expect("read local runtime inventory"); + assert_eq!(info.disks.len(), drive_count, "offline members retain their topology slots"); + assert_eq!( + storage_read_ready_from_runtime_state(&info), + survivors >= data, + "layout={drive_count}/{parity}, survivors={survivors}" + ); + assert_eq!( + storage_ready_from_runtime_state(&info), + survivors >= write_quorum, + "layout={drive_count}/{parity}, survivors={survivors}" + ); + } + } + } + + #[test] + fn node_disk_snapshot_requires_online_runtime_before_counting_recovered_drives() { + for is_local in [false, true] { + for reachable in [false, true] { + let online_hosts = if reachable { + HashSet::from(["node-0:9000".to_string()]) + } else { + HashSet::new() + }; + for runtime_state in ["online", "suspect", "offline", "returning", "unknown"] { + let endpoint = Endpoint { + url: url::Url::parse("http://node-0:9000/data").expect("valid test endpoint"), + is_local, + pool_idx: 0, + set_idx: 0, + disk_idx: 2, + }; + let mut disks = online_readiness_disks(0, 2); + disks.push(node_disk_snapshot(endpoint, runtime_state, &online_hosts)); + let info = StorageInfo { + backend: BackendInfo { + total_sets: vec![1], + drives_per_set: vec![4], + standard_sc_data: vec![2], + standard_sc_parities: vec![2], + ..Default::default() + }, + disks, + ..Default::default() + }; + assert!(storage_read_ready_from_runtime_state(&info)); + assert_eq!( + storage_ready_from_runtime_state(&info), + runtime_state == "online" && (is_local || reachable), + "runtime_state={runtime_state}, local={is_local}, reachable={reachable}" + ); + } + } + } + } + + #[tokio::test] + async fn node_storage_snapshot_requires_each_configured_set_and_distinct_drives() { + let mut store = runtime_inventory(&[(2, 4, 2), (1, 8, 2)]).await; + let online_hosts = (0..8).map(|idx| format!("node-{idx}:9000")).collect(); + let info = node_storage_snapshot(&store, &online_hosts) + .await + .expect("healthy mixed layout"); + assert!(storage_ready_from_runtime_state(&info)); + + let selector = DiskSetSelector::new(0, 1); + let original = store.disks.remove(&selector).expect("second configured set"); + let info = node_storage_snapshot(&store, &online_hosts) + .await + .expect("missing set snapshot"); + assert!(!storage_read_ready_from_runtime_state(&info)); + assert!(!storage_ready_from_runtime_state(&info)); + + store.disks.insert(selector, vec![original[0].clone(); 4]); + let info = node_storage_snapshot(&store, &online_hosts) + .await + .expect("duplicate drive snapshot"); + assert!(!storage_read_ready_from_runtime_state(&info)); + assert!(!storage_ready_from_runtime_state(&info)); + + store.disks.insert(selector, original); + assert!(storage_ready_from_runtime_state( + &node_storage_snapshot(&store, &online_hosts) + .await + .expect("restored inventory") + )); + } + + #[tokio::test] + async fn node_storage_snapshot_bounds_waits_and_propagates_inventory_failure() { + let mut store = runtime_inventory(&[(1, 4, 2)]).await; + let online_hosts = (0..4).map(|idx| format!("node-{idx}:9000")).collect(); + let selector = DiskSetSelector::new(0, 0); + store.pending_set = Some(selector); + let result = tokio::time::timeout(Duration::from_secs(1), node_storage_snapshot(&store, &online_hosts)) + .await + .expect("node storage inspection must honor its own 100 ms budget"); + assert!(matches!(result, Err(StorageError::Timeout))); + + store.pending_set = None; + store.failed_set = Some(selector); + assert!(node_storage_snapshot(&store, &online_hosts).await.is_err()); + store.failed_set = None; + assert!(storage_ready_from_runtime_state( + &node_storage_snapshot(&store, &online_hosts) + .await + .expect("inspection recovered") + )); + } + fn online_readiness_disks(set_idx: i32, count: i32) -> Vec { (0..count) .map(|disk_index| Disk { @@ -1169,6 +1474,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, }; let first_calls = calls.clone(); @@ -1225,6 +1531,7 @@ mod tests { peer_health_ready: true, }, degraded_reasons: Vec::new(), + storage_details: None, } }) .await; @@ -2025,24 +2332,17 @@ mod tests { *guard = None; } - update_lock_quorum_status_cache(LockQuorumStatus { - ready: true, - connected_clients: 2, - total_clients: 3, - required_quorum: 2, - }) - .await; - - let cached = load_cached_lock_quorum_status().await; - assert_eq!( - cached, - Some(LockQuorumStatus { + let observation = LockQuorumObservation { + status: LockQuorumStatus { ready: true, connected_clients: 2, total_clients: 3, required_quorum: 2, - }) - ); + }, + online_hosts: HashSet::from(["node-a:9000".to_owned(), "node-b:9000".to_owned()]), + }; + update_lock_quorum_status_cache(observation.clone()).await; + assert_eq!(load_cached_lock_quorum_status().await, Some(observation)); }) .await; } diff --git a/rustfs/src/server/ssec_transport.rs b/rustfs/src/server/ssec_transport.rs new file mode 100644 index 000000000..cb65b258a --- /dev/null +++ b/rustfs/src/server/ssec_transport.rs @@ -0,0 +1,276 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! SSE-C transport policy (backlog#2369 P7.2). +//! +//! An SSE-C request carries the customer's AES key in a request header, so AWS +//! S3 and MinIO both refuse one that did not arrive over TLS. RustFS accepted +//! them on any transport, which means a plaintext hop hands the key to anyone +//! on the path — and the object is then unreadable without that same key, so +//! the exposure is permanent for as long as the object lives. +//! +//! Refusing outright is the correct end state but not a safe default to adopt +//! inside a release window: the project's own s3-tests and e2e lanes, and most +//! staging deployments, speak plain HTTP. This release therefore reports: +//! every SSE-C request on a plaintext transport increments +//! `rustfs_ssec_plaintext_requests_total` and logs one warning per process, so +//! an operator can see whether anything would break before the default flips. +//! `RUSTFS_SSE_C_REQUIRE_TLS=true` opts a deployment into the rejection now. +//! +//! The transport verdict is per connection, not per deployment: the layer is +//! built with whether *this* listener terminates TLS, and additionally accepts +//! a `https` forwarded protocol resolved by the trusted-proxy layer, which is +//! the only spoof-resistant source for a TLS-terminating proxy in front. + +use bytes::Bytes; +use futures::future::{Either, Ready, ready}; +use http::{HeaderMap, HeaderValue, Request, Response, StatusCode}; +use http_body_util::{BodyExt, Full}; +use metrics::counter; +use rustfs_trusted_proxies::ClientInfo; +use rustfs_utils::http::headers::{ + AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, + AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, +}; +use std::sync::Once; +use std::task::{Context, Poll}; +use tower::{Layer, Service}; +use tracing::warn; + +use crate::storage_api::server::layer::request_context::RequestContext; + +/// Opt in to refusing SSE-C on a plaintext transport. Default `false` for this +/// release; the reporting path runs either way. +pub(crate) const ENV_SSE_C_REQUIRE_TLS: &str = "RUSTFS_SSE_C_REQUIRE_TLS"; +pub(crate) const DEFAULT_SSE_C_REQUIRE_TLS: bool = false; + +/// Counts SSE-C requests that arrived without TLS. A deployment planning to +/// enable [`ENV_SSE_C_REQUIRE_TLS`] should see this at zero first. +pub(crate) const METRIC_SSEC_PLAINTEXT_REQUESTS_TOTAL: &str = "rustfs_ssec_plaintext_requests_total"; + +type BoxError = Box; +type BoxBody = http_body_util::combinators::UnsyncBoxBody; + +/// Whether the request carries any SSE-C header. +/// +/// Any one of the three is enough: an incomplete triple is still an attempt to +/// use SSE-C, and it is rejected later for being incomplete — but the key may +/// already have crossed the wire. +fn carries_ssec_headers(headers: &HeaderMap) -> bool { + headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM) + || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY) + || headers.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5) +} + +/// Whether this request reached the server over TLS. +/// +/// `connection_is_tls` is what this listener actually did. The forwarded +/// protocol is only consulted as a second source because the trusted-proxy +/// layer has already decided whether the peer is allowed to assert it; a +/// request that arrives direct carries no such assertion. +fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool { + if connection_is_tls { + return true; + } + client_info + .and_then(|info| info.forwarded_proto.as_deref()) + .is_some_and(|proto| proto.eq_ignore_ascii_case("https")) +} + +fn require_tls() -> bool { + rustfs_utils::get_env_bool(ENV_SSE_C_REQUIRE_TLS, DEFAULT_SSE_C_REQUIRE_TLS) +} + +/// One warning per process: plaintext SSE-C traffic is driven by clients, so a +/// per-request warning would let a busy client flood the log. The counter +/// carries the per-request volume. +fn warn_once_about_plaintext_ssec() { + static WARNED: Once = Once::new(); + WARNED.call_once(|| { + warn!( + event = "ssec_request_without_tls", + require_tls = ENV_SSE_C_REQUIRE_TLS, + metric = METRIC_SSEC_PLAINTEXT_REQUESTS_TOTAL, + "SSE-C requests are arriving without TLS, so the customer key crosses the network in \ + cleartext. AWS S3 refuses these; RustFS will too in a later release. Terminate TLS on \ + this listener or on a trusted proxy, then set RUSTFS_SSE_C_REQUIRE_TLS=true. Reported \ + once per process; the counter carries the volume." + ); + }); +} + +/// The S3 rejection AWS returns for SSE-C without TLS. Built by hand because +/// the rejection short-circuits the inner response stack, mirroring the +/// rate-limit layer. +fn ssec_requires_tls_response(request_id: Option<&str>) -> Response { + let request_id_xml = request_id + .filter(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')) + .map(|id| format!("{id}")) + .unwrap_or_default(); + let body = format!( + "\ + InvalidRequest\ + Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.\ + {request_id_xml}" + ); + let body: BoxBody = Full::new(Bytes::from(body)) + .map_err(|e| -> BoxError { Box::new(e) }) + .boxed_unsync(); + + let mut response = Response::new(body); + *response.status_mut() = StatusCode::BAD_REQUEST; + response + .headers_mut() + .insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/xml")); + response +} + +/// Layer that reports — and optionally refuses — SSE-C over a plaintext +/// transport. `connection_is_tls` is whether the listener that accepted this +/// connection terminated TLS. +#[derive(Clone, Copy)] +pub(crate) struct SsecTransportLayer { + connection_is_tls: bool, +} + +impl SsecTransportLayer { + pub(crate) fn new(connection_is_tls: bool) -> Self { + Self { connection_is_tls } + } +} + +impl Layer for SsecTransportLayer { + type Service = SsecTransportService; + + fn layer(&self, inner: S) -> Self::Service { + SsecTransportService { + inner, + connection_is_tls: self.connection_is_tls, + } + } +} + +#[derive(Clone)] +pub(crate) struct SsecTransportService { + inner: S, + connection_is_tls: bool, +} + +impl Service> for SsecTransportService +where + S: Service, Response = Response>, +{ + type Response = Response; + type Error = S::Error; + type Future = Either, S::Error>>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + if !carries_ssec_headers(req.headers()) + || is_secure_transport(self.connection_is_tls, req.extensions().get::()) + { + return Either::Left(self.inner.call(req)); + } + + counter!(METRIC_SSEC_PLAINTEXT_REQUESTS_TOTAL).increment(1); + warn_once_about_plaintext_ssec(); + + if !require_tls() { + return Either::Left(self.inner.call(req)); + } + + Either::Right(ready(Ok(ssec_requires_tls_response( + req.extensions() + .get::() + .map(|context| context.request_id.as_str()), + )))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, SocketAddr}; + + fn ssec_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, HeaderValue::from_static("AES256")); + headers + } + + fn proxied(proto: &str) -> ClientInfo { + ClientInfo::from_trusted_proxy( + IpAddr::from([203, 0, 113, 10]), + None, + Some(proto.to_string()), + IpAddr::from([10, 0, 0, 1]), + 1, + rustfs_trusted_proxies::ValidationMode::Lenient, + Vec::new(), + ) + } + + fn direct() -> ClientInfo { + ClientInfo::direct(SocketAddr::new(IpAddr::from([203, 0, 113, 10]), 443)) + } + + #[test] + fn any_ssec_header_counts_as_an_ssec_request() { + assert!(!carries_ssec_headers(&HeaderMap::new())); + assert!(carries_ssec_headers(&ssec_headers())); + + // An incomplete triple is still an attempt, and the key may already + // have crossed the wire. + let mut only_key = HeaderMap::new(); + only_key.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, HeaderValue::from_static("a2V5")); + assert!(carries_ssec_headers(&only_key)); + + let mut only_md5 = HeaderMap::new(); + only_md5.insert(AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, HeaderValue::from_static("bWQ1")); + assert!(carries_ssec_headers(&only_md5)); + } + + #[test] + fn transport_is_secure_only_on_tls_or_a_resolved_https_proxy() { + assert!(is_secure_transport(true, None), "a TLS listener needs no header to prove it"); + assert!(is_secure_transport(true, Some(&proxied("http"))), "the listener's own TLS wins"); + assert!( + is_secure_transport(false, Some(&proxied("https"))), + "a TLS-terminating trusted proxy is a secure transport" + ); + assert!( + !is_secure_transport(false, Some(&proxied("http"))), + "a proxy that forwarded plain HTTP is not" + ); + assert!( + !is_secure_transport(false, Some(&direct())), + "a direct plaintext client asserts no protocol" + ); + assert!(!is_secure_transport(false, None), "no transport evidence means not secure"); + } + + #[test] + fn the_rejection_carries_the_aws_wording_and_a_safe_request_id() { + let response = ssec_requires_tls_response(Some("abc-123")); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + // A request id that is not plain enough to embed must be dropped + // rather than escaped into the XML body. + let injected = ssec_requires_tls_response(Some("")); + assert_eq!(injected.status(), StatusCode::BAD_REQUEST); + } +} diff --git a/rustfs/src/shared_types.rs b/rustfs/src/shared_types.rs index 7406a0978..3a872eb8c 100644 --- a/rustfs/src/shared_types.rs +++ b/rustfs/src/shared_types.rs @@ -45,6 +45,7 @@ pub enum ReadinessDegradedReason { ObjectWriteStalled, PoolMetaWriteBlocked, PoolMetadataCheckTimeout, + StorageReadinessCheckTimeout, ClusterHealthTimeout, PeerHealthUnavailable, StartupFinalizationPending, @@ -65,6 +66,7 @@ impl ReadinessDegradedReason { ReadinessDegradedReason::ObjectWriteStalled => "object_write_stalled", ReadinessDegradedReason::PoolMetaWriteBlocked => "pool_meta_write_blocked", ReadinessDegradedReason::PoolMetadataCheckTimeout => "pool_metadata_check_timeout", + ReadinessDegradedReason::StorageReadinessCheckTimeout => "storage_readiness_check_timeout", ReadinessDegradedReason::ClusterHealthTimeout => "cluster_health_timeout", ReadinessDegradedReason::PeerHealthUnavailable => "peer_health_unavailable", ReadinessDegradedReason::StartupFinalizationPending => "startup_finalization_pending", @@ -80,6 +82,14 @@ impl ReadinessDegradedReason { pub struct DependencyReadinessReport { pub readiness: DependencyReadiness, pub degraded_reasons: Vec, + pub storage_details: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StorageReadinessDetails { + pub read_quorum_ready: bool, + pub write_quorum_ready: bool, + pub pool_metadata_write_ready: bool, } pub(crate) fn convert_ecstore_object_info(object: StorageObjectInfo) -> NotifyObjectInfo { diff --git a/rustfs/src/startup_shutdown.rs b/rustfs/src/startup_shutdown.rs index 70b1a0701..50f855662 100644 --- a/rustfs/src/startup_shutdown.rs +++ b/rustfs/src/startup_shutdown.rs @@ -280,6 +280,7 @@ pub(crate) async fn run_startup_shutdown_sequence( let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true); let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true); + let mut heal_handoff_complete = true; let background_steps = background_shutdown_steps(enable_scanner, enable_heal); for step in &background_steps { match step { @@ -305,7 +306,19 @@ pub(crate) async fn run_startup_shutdown_sequence( state = "stopping", "Background service shutdown started" ); - shutdown_ahm_services(); + if let Err(error) = shutdown_ahm_services().await { + heal_handoff_complete = false; + warn!( + target: "rustfs::main::handle_shutdown", + event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + service = "ahm", + state = "handoff_failed", + error = %error, + "Heal shutdown handoff failed; retaining unclean-shutdown markers" + ); + } } } } @@ -411,7 +424,9 @@ pub(crate) async fn run_startup_shutdown_sequence( shutdown_optional_runtime_services(optional_runtime_shutdowns).await; // The data plane is drained: record this shutdown as clean so the next // startup skips the unclean-restart erasure-set heal. - rustfs_heal::heal::clear_unclean_shutdown_markers().await; + if heal_handoff_complete { + rustfs_heal::heal::clear_unclean_shutdown_markers().await; + } state_manager.update(ServiceState::Stopped); info!( target: "rustfs::main::handle_shutdown", diff --git a/rustfs/src/startup_storage.rs b/rustfs/src/startup_storage.rs index d25fb794b..93341b9a4 100644 --- a/rustfs/src/startup_storage.rs +++ b/rustfs/src/startup_storage.rs @@ -152,6 +152,7 @@ pub(crate) async fn init_startup_storage_runtime( readiness: Arc, instance_ctx: Arc, ) -> Result { + rustfs_scanner::register_scanner_pause_backlog_retirement(); let ctx = CancellationToken::new(); debug!( @@ -195,6 +196,7 @@ pub(crate) async fn init_embedded_startup_storage_runtime( shutdown_token: CancellationToken, instance_ctx: Arc, ) -> Result { + rustfs_scanner::register_scanner_pause_backlog_retirement(); let store = match ECStore::new_with_instance_ctx(server_addr, endpoint_pools.clone(), shutdown_token.clone(), instance_ctx).await { Ok(store) => store, diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index a2a95b017..405be6281 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -1538,6 +1538,14 @@ fn table_catalog_store_for_data_plane( .map_err(|err| s3_error!(InternalError, "failed to configure table catalog backing: {}", err)) } +fn table_publication_guard_error(err: crate::table_catalog::TableCatalogStoreError) -> S3Error { + let code = match &err { + crate::table_catalog::TableCatalogStoreError::Unavailable(_) => S3ErrorCode::ServiceUnavailable, + _ => S3ErrorCode::InternalError, + }; + S3Error::with_message(code, format!("failed to acquire table publication guard: {err}")) +} + async fn retain_table_data_plane_publication_guard( req: &mut S3Request, table_bucket: &str, @@ -1556,7 +1564,7 @@ async fn retain_table_data_plane_publication_guard( let backend = table_catalog_backend_for_data_plane(req)?; let guard = crate::table_catalog::TableCatalogObjectBackend::acquire_read_lock(&backend, table_bucket, lock_object) .await - .map_err(|err| s3_error!(InternalError, "failed to acquire table publication guard: {}", err))?; + .map_err(table_publication_guard_error)?; let mut state = retained.state.lock(); state.keys.insert(key); state.guards.push(Box::new(guard)); @@ -3106,7 +3114,8 @@ mod tests { merge_request_object_tag_conditions, owner_can_bypass_policy_deny, post_object_authorize_action, put_bucket_policy_authorize_action, request_context_from_req, request_object_store, retention_write_requested, secondary_tag_hint_action, table_data_plane_admin_action, table_data_plane_content_mutation, - table_data_plane_resource_for_request, validate_post_object_success_controls, versioned_read_action, + table_data_plane_resource_for_request, table_publication_guard_error, validate_post_object_success_controls, + versioned_read_action, }; use crate::error::ApiError; use crate::storage::storage_api::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; @@ -3268,6 +3277,155 @@ mod tests { } } + #[test] + fn table_publication_guard_unavailable_errors_are_retryable() { + use crate::table_catalog::TableCatalogStoreError; + + for message in [ + "failed to acquire catalog migration lock: Quorum not reached: required 2, achieved 1", + "failed to acquire catalog migration lock: lock acquisition timed out after 5s", + "peer unavailable", + "", + ] { + let error = TableCatalogStoreError::Unavailable(message.to_string()); + let expected_message = format!("failed to acquire table publication guard: {error}"); + let err = table_publication_guard_error(error); + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "{message}"); + assert_eq!( + err.status_code().or_else(|| err.code().status_code()), + Some(http::StatusCode::SERVICE_UNAVAILABLE) + ); + assert_eq!(err.message(), Some(expected_message.as_str())); + } + } + + #[test] + fn table_publication_guard_non_retryable_errors_stay_internal() { + use crate::table_catalog::TableCatalogStoreError; + + let message = "temporarily unavailable: timeout: quorum not reached".to_string(); + for error in [ + TableCatalogStoreError::Internal(message.clone()), + TableCatalogStoreError::Invalid(message.clone()), + TableCatalogStoreError::Unsupported(message.clone()), + TableCatalogStoreError::NotFound(message.clone()), + TableCatalogStoreError::NamespaceNotFound(message.clone()), + TableCatalogStoreError::TableNotFound(message.clone()), + TableCatalogStoreError::AlreadyExists(message.clone()), + TableCatalogStoreError::Conflict(message), + ] { + let expected_message = format!("failed to acquire table publication guard: {error}"); + let err = table_publication_guard_error(error); + assert_eq!(err.code(), &S3ErrorCode::InternalError); + assert_eq!( + err.status_code().or_else(|| err.code().status_code()), + Some(http::StatusCode::INTERNAL_SERVER_ERROR) + ); + assert_eq!(err.message(), Some(expected_message.as_str())); + } + } + + #[tokio::test] + #[serial] + async fn table_publication_guard_timeout_blocks_put_and_multipart_authorization() { + use crate::storage::storage_api::contract::namespace::NamespaceLocking as _; + use std::time::Duration; + + let store = crate::app::gating_test_env::shared_gating_ecstore().await; + let server_ctx = ServerContextSlot::new(); + assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms))))); + let fs = FS::with_server_ctx(server_ctx); + let bucket = format!("publication-timeout-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("test bucket should be created"); + let policy_json = format!( + r#"{{"Version":"2012-10-17","Statement":[{{"Effect":"Allow","Principal":{{"AWS":"*"}},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::{bucket}/*"]}}]}}"# + ); + let mut metadata = (*crate::storage::get_bucket_metadata(&bucket) + .await + .expect("test bucket metadata should be cached")) + .clone(); + metadata.policy_config = Some(serde_json::from_str(&policy_json).expect("test policy should parse")); + metadata.policy_config_json = policy_json.into_bytes(); + crate::storage::storage_api::set_bucket_metadata(bucket.clone(), metadata) + .await + .expect("test policy should be published"); + let lock_object = crate::table_catalog::default_table_bucket_publication_lock_path(); + let lock = store + .new_ns_lock(&bucket, &lock_object) + .await + .expect("publication lock should be created"); + let mut writer = lock + .get_write_lock(Duration::from_secs(1)) + .await + .expect("publication writer should hold the fence"); + let mut put_req = build_request( + PutObjectInput::builder() + .bucket(bucket.clone()) + .key("object".to_string()) + .build() + .expect("PUT input should build"), + Method::PUT, + ); + let mut multipart_req = build_request( + CreateMultipartUploadInput::builder() + .bucket(bucket.clone()) + .key("multipart-object".to_string()) + .build() + .expect("multipart input should build"), + Method::POST, + ); + ensure_req_info(&mut put_req); + ensure_req_info(&mut multipart_req); + put_req.extensions.insert(fs.server_ctx().clone()); + multipart_req.extensions.insert(fs.server_ctx().clone()); + + for (operation, err) in [ + ( + "PutObject", + fs.put_object(&mut put_req) + .await + .expect_err("PUT must wait for the publication writer"), + ), + ( + "CreateMultipartUpload", + fs.create_multipart_upload(&mut multipart_req) + .await + .expect_err("multipart initialization must wait for the publication writer"), + ), + ] { + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "{operation}: {err}"); + } + for extensions in [&put_req.extensions, &multipart_req.extensions] { + assert!(extensions.get::().is_none()); + } + + assert!(writer.release()); + for _ in 0..2 { + fs.put_object(&mut put_req) + .await + .expect("PUT may retry after publication finishes"); + fs.create_multipart_upload(&mut multipart_req) + .await + .expect("multipart initialization may retry after publication finishes"); + for extensions in [&put_req.extensions, &multipart_req.extensions] { + let retained = extensions + .get::() + .expect("successful admission retains the guard"); + let state = retained.state.lock(); + assert!(state.keys.contains(&(bucket.clone(), lock_object.clone()))); + assert_eq!(state.guards.len(), 1, "repeated admission must reuse its retained guard"); + } + } + drop(put_req); + drop(multipart_req); + lock.get_write_lock(Duration::from_secs(1)) + .await + .expect("dropping the request must release its publication guard"); + } + #[test] fn table_data_plane_mutations_fence_before_table_bucket_marker_lookup() { let source = include_str!("access.rs"); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index e62cd6d82..93cbc5ee9 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -730,7 +730,11 @@ impl S3 for FS { let result = Ok(S3Response::new(DeleteObjectTaggingOutput { version_id })); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &object, + rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata, + ); let duration = start_time.elapsed(); histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "delete").record(duration.as_secs_f64()); result @@ -1629,7 +1633,11 @@ impl S3 for FS { let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata, + ); result } @@ -1733,7 +1741,10 @@ impl S3 for FS { ); } - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_bucket_from_producer( + &bucket, + rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata, + ); Ok(S3Response::new(PutObjectLockConfigurationOutput::default())) } @@ -1849,7 +1860,11 @@ impl S3 for FS { let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &key, + rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata, + ); result } @@ -1959,7 +1974,11 @@ impl S3 for FS { version_id: req.input.version_id.clone(), })); let _ = helper.complete(&result); - rustfs_scanner::record_dirty_usage_bucket(&bucket); + rustfs_scanner::record_dirty_usage_object_from_producer( + &bucket, + &object, + rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata, + ); let duration = start_time.elapsed(); histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "put").record(duration.as_secs_f64()); result diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 9ecd207a9..c3807527e 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -1006,12 +1006,22 @@ pub(crate) async fn apply_cors_headers(bucket: &str, method: &http::Method, head } // Access-Control-Allow-Headers (required for preflight if headers were requested) - if is_preflight && let Some(ref allowed_headers) = rule.allowed_headers { - let headers_str = allowed_headers.iter().map(|h| h.as_str()).collect::>().join(", "); + if is_preflight && let Some(ref requested_headers) = requested_headers { + // Every requested header matched this rule; do not expose its wildcard + // or grant headers that the preflight did not request. + let headers_str = requested_headers.join(","); if let Ok(headers_value) = HeaderValue::from_str(&headers_str) { response_headers.insert(cors::response::ACCESS_CONTROL_ALLOW_HEADERS, headers_value); } } + if is_preflight { + let vary = if origin_reflected { + "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" + } else { + "Access-Control-Request-Method, Access-Control-Request-Headers" + }; + response_headers.insert(cors::standard::VARY, HeaderValue::from_static(vary)); + } // Access-Control-Expose-Headers (for actual requests) if !is_preflight && let Some(ref expose_headers) = rule.expose_headers { diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index cf4f011a4..87852378c 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -1736,7 +1736,10 @@ mod tests { "https://console.localhost", ); assert_eq!(result.get(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS).unwrap(), "true"); - assert_eq!(result.get(cors::standard::VARY).unwrap(), "Origin"); + assert_eq!( + result.get(cors::standard::VARY).unwrap(), + "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" + ); set_bucket_metadata(bucket.to_string(), BucketMetadata::new(bucket)) .await diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index f1fe52979..af98b5bb3 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -38,6 +38,7 @@ mod minio_generated_read_test; #[cfg(test)] mod multi_factor_scheduler_integration_test; pub(crate) mod runtime_sources; +pub(crate) mod scanner_dirty_journal; #[cfg(test)] mod sse_test; pub(crate) mod storage_api; diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 306933145..4ba5f9461 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -182,10 +182,10 @@ fn remove_heal_control_replay( static HEAL_CONTROL_REPLAY_CACHE: OnceLock>>> = OnceLock::new(); static NODE_CAPABILITY_SERVER_EPOCH: LazyLock = LazyLock::new(Uuid::new_v4); // v3 additionally promises the v6 tier-delete dispatch-manifest policy; v4 -// promises the sticky per-target decommission capacity fence. The -// existing periodic topology probe carries both capabilities so normal object -// operations do not add another peer RPC. -const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 4; +// promises the sticky per-target decommission capacity fence; v5 supports +// conditional transition-state repair and preserves its destination binding. +// Normal object operations reuse the periodic topology capability probe. +const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 5; fn encode_heal_capability_response( topology_member: &str, @@ -3100,7 +3100,7 @@ mod tests { rustfs_heal_contracts::heal_channel::HealChannelRequest, rustfs_protos::heal_control::RequestMetadata, ) { - let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None)); + let manager = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(HealControlMockStorage), None)); let mut request = rustfs_heal_contracts::heal_channel::create_heal_request( "bucket".to_string(), Some("prefix".to_string()), @@ -3338,8 +3338,8 @@ mod tests { } if task_id == next_id && task_id != first_id)); assert_eq!( manager.operations_snapshot().await.queue_length, - 2, - "a caller must not treat a new forced request as an idempotent transport retry" + 1, + "a fresh forceStart should replace the previous same-target queued task" ); } @@ -3475,6 +3475,9 @@ mod tests { 1, "post-admission response loss must leave exactly one canonical task" ); + if let Some(cache) = super::HEAL_CONTROL_REPLAY_CACHE.get() { + cache.lock().await.clear(); + } let mut retry = connect_faulty_heal_control_client( Arc::clone(&manager), @@ -3554,8 +3557,8 @@ mod tests { )); assert_eq!( manager.operations_snapshot().await.queue_length, - 2, - "a new forceStart request must be counted as a distinct canonical task" + 1, + "a fresh forceStart should replace the previous same-target queued task" ); } @@ -3607,7 +3610,7 @@ mod tests { } if task_id == first_id )); - let restarted_manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None)); + let restarted_manager = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(HealControlMockStorage), None)); let mut restarted_peer = connect_faulty_heal_control_client( Arc::clone(&restarted_manager), fingerprint, @@ -3660,7 +3663,7 @@ mod tests { #[tokio::test] async fn heal_control_executor_preserves_canonical_token_and_drops_query_results() { - let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None)); + let manager = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(HealControlMockStorage), None)); let coordinator_epoch = 7; let now = OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000; let now = i64::try_from(now).expect("test clock should fit in i64"); diff --git a/rustfs/src/storage/scanner_dirty_journal.rs b/rustfs/src/storage/scanner_dirty_journal.rs new file mode 100644 index 000000000..5ab3a1662 --- /dev/null +++ b/rustfs/src/storage/scanner_dirty_journal.rs @@ -0,0 +1,597 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ECStore, Error, ecstore_config}; +use rustfs_scanner::{ + ScannerDirtyUsageBucket, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayRecord, + ScannerDurableDirtyUsageReplayScope, SegmentInvalidationProducerIdentity, +}; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tracing::{debug, warn}; + +const LOG_COMPONENT_STORAGE: &str = "storage"; +const LOG_SUBSYSTEM_SCANNER: &str = "scanner"; +const EVENT_SCANNER_DIRTY_USAGE_JOURNAL: &str = "scanner_dirty_usage_journal"; +const DURABLE_DIRTY_USAGE_REPLAY_OBJECT: &str = "scanner/durable-dirty-producer-replay.json"; +const DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES: usize = 64 * 1024; +const DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES: usize = 1024; +const DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES: usize = 128; +const DURABLE_DIRTY_USAGE_JOURNAL_CHANNEL_DEPTH: usize = 256; +const DURABLE_DIRTY_USAGE_JOURNAL_RETRY_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Clone)] +pub(crate) struct DurableDirtyUsageJournal { + sender: mpsc::Sender, +} + +impl DurableDirtyUsageJournal { + pub(crate) fn record_committed_mutation(&self, bucket: &str, object: &str, producer: SegmentInvalidationProducerIdentity) { + let snapshot = rustfs_scanner::scanner_dirty_usage_snapshot(DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES); + let generation = snapshot + .buckets + .into_iter() + .find(|entry| entry.bucket == bucket) + .map(|entry| entry.generation) + .unwrap_or(0); + self.dispatch(JournalCommand::RecordMutation { + bucket: bucket.to_string(), + object: object.to_string(), + generation, + producer, + }); + } + + pub(crate) fn clear_confirmed_buckets(&self, cleared: Vec) { + self.dispatch(JournalCommand::ClearBuckets { cleared }); + } + + fn dispatch(&self, command: JournalCommand) { + match self.sender.try_send(command) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(command)) => { + let sender = self.sender.clone(); + tokio::spawn(async move { + let _ = sender.send(command).await; + }); + } + Err(mpsc::error::TrySendError::Closed(_)) => {} + } + } +} + +#[derive(Debug)] +enum JournalCommand { + RecordMutation { + bucket: String, + object: String, + generation: u64, + producer: SegmentInvalidationProducerIdentity, + }, + ClearBuckets { + cleared: Vec, + }, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct JournalState { + buckets: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct JournalBucketState { + generation: u64, + scope: JournalScope, + producers: BTreeSet, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum JournalScope { + WholeBucket, + TopLevelEntries(BTreeSet), +} + +impl JournalState { + fn record_mutation(&mut self, bucket: String, object: &str, generation: u64, producer: SegmentInvalidationProducerIdentity) { + if bucket.is_empty() || generation == 0 || generation == u64::MAX || !durable_dirty_usage_producer_is_supported(producer) + { + self.buckets.clear(); + return; + } + + let event_scope = durable_dirty_usage_journal_scope(object); + self.buckets + .entry(bucket) + .and_modify(|state| { + state.generation = state.generation.max(generation); + merge_journal_scope(&mut state.scope, event_scope.clone()); + state.producers.insert(producer); + }) + .or_insert_with(|| JournalBucketState { + generation, + scope: event_scope, + producers: BTreeSet::from([producer]), + }); + } + + fn clear_buckets(&mut self, cleared: &[ScannerDirtyUsageBucket]) { + for entry in cleared { + if self + .buckets + .get(&entry.bucket) + .is_some_and(|state| state.generation <= entry.generation) + { + self.buckets.remove(&entry.bucket); + } + } + } + + fn replay_entries(&self) -> Option> { + if self.buckets.is_empty() || self.buckets.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES { + return None; + } + let mut entries = Vec::with_capacity(self.buckets.len()); + for (bucket, state) in &self.buckets { + if state.producers.is_empty() { + return None; + } + let scope = match &state.scope { + JournalScope::WholeBucket => ScannerDurableDirtyUsageReplayScope::WholeBucket, + JournalScope::TopLevelEntries(entries) => { + if entries.is_empty() || entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES { + return None; + } + ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: entries.clone(), + } + } + }; + entries.push(ScannerDurableDirtyUsageReplayEntry { + bucket: bucket.clone(), + generation: state.generation, + scope, + producers: state.producers.clone(), + }); + } + Some(entries) + } + + fn from_replay_record(record: ScannerDurableDirtyUsageReplayRecord) -> Option { + if record.entries.is_empty() || record.entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_ENTRIES { + return None; + } + let mut state = JournalState::default(); + for entry in record.entries { + if entry.bucket.is_empty() + || entry.bucket.contains(['/', '\\', '\0']) + || entry.bucket == "." + || entry.bucket == ".." + || entry.generation == 0 + || entry.generation == u64::MAX + || entry.producers.is_empty() + || entry + .producers + .iter() + .any(|producer| !durable_dirty_usage_producer_is_supported(*producer)) + { + return None; + } + let scope = match entry.scope { + ScannerDurableDirtyUsageReplayScope::WholeBucket => JournalScope::WholeBucket, + ScannerDurableDirtyUsageReplayScope::TopLevelEntries { entries } => { + if entries.is_empty() || entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES { + return None; + } + if entries + .iter() + .any(|entry| durable_dirty_usage_top_level_entry(entry).as_deref() != Some(entry.as_str())) + { + return None; + } + JournalScope::TopLevelEntries(entries) + } + }; + if state + .buckets + .insert( + entry.bucket, + JournalBucketState { + generation: entry.generation, + scope, + producers: entry.producers, + }, + ) + .is_some() + { + return None; + } + } + Some(state) + } +} + +pub(crate) async fn start_durable_dirty_usage_journal(store: Arc) -> DurableDirtyUsageJournal { + let initial_state = replay_durable_dirty_usage_journal(store.clone()).await; + let (sender, receiver) = mpsc::channel(DURABLE_DIRTY_USAGE_JOURNAL_CHANNEL_DEPTH); + tokio::spawn(run_durable_dirty_usage_journal(store, receiver, initial_state)); + DurableDirtyUsageJournal { sender } +} + +async fn replay_durable_dirty_usage_journal(store: Arc) -> JournalState { + let bytes = match super::read_config(store, DURABLE_DIRTY_USAGE_REPLAY_OBJECT).await { + Ok(bytes) if !bytes.is_empty() => bytes, + Ok(_) | Err(Error::ConfigNotFound) => return JournalState::default(), + Err(err) => { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "read_failed", + error = ?err, + "Durable scanner dirty usage journal could not be read" + ); + return JournalState::default(); + } + }; + if bytes.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_BYTES { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "oversized", + "Durable scanner dirty usage journal exceeds replay size limit" + ); + return JournalState::default(); + } + match rustfs_scanner::replay_durable_dirty_usage_producer_record(&bytes) { + Ok(state) => { + debug!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "replayed", + generation = state.generation, + pending = state.pending, + "Durable scanner dirty usage journal replayed" + ); + match serde_json::from_slice::(&bytes) + .ok() + .and_then(JournalState::from_replay_record) + { + Some(state) => state, + None => { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "hydrate_rejected", + "Durable scanner dirty usage journal could not hydrate writer state" + ); + JournalState::default() + } + } + } + Err(err) => { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "replay_rejected", + error = ?err, + "Durable scanner dirty usage journal was rejected" + ); + JournalState::default() + } + } +} + +async fn run_durable_dirty_usage_journal( + store: Arc, + mut receiver: mpsc::Receiver, + initial_state: JournalState, +) { + let mut state = initial_state; + let mut dirty = false; + let mut retry = tokio::time::interval(DURABLE_DIRTY_USAGE_JOURNAL_RETRY_INTERVAL); + retry.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + tokio::select! { + command = receiver.recv() => { + let Some(command) = command else { + break; + }; + apply_journal_command(&mut state, command); + while let Ok(command) = receiver.try_recv() { + apply_journal_command(&mut state, command); + } + dirty = true; + } + _ = retry.tick(), if dirty => {} + } + + if dirty && flush_durable_dirty_usage_journal(store.clone(), &state).await { + dirty = false; + } + } +} + +fn apply_journal_command(state: &mut JournalState, command: JournalCommand) { + match command { + JournalCommand::RecordMutation { + bucket, + object, + generation, + producer, + } => state.record_mutation(bucket, &object, generation, producer), + JournalCommand::ClearBuckets { cleared } => state.clear_buckets(&cleared), + } +} + +async fn flush_durable_dirty_usage_journal(store: Arc, state: &JournalState) -> bool { + let Some(entries) = state.replay_entries() else { + return delete_durable_dirty_usage_journal(store).await; + }; + let bytes = match rustfs_scanner::encode_durable_dirty_usage_producer_replay_record(entries) { + Ok(bytes) => bytes, + Err(err) => { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "encode_rejected", + error = ?err, + "Durable scanner dirty usage journal could not be encoded" + ); + return delete_durable_dirty_usage_journal(store).await; + } + }; + match ecstore_config::com::save_config(store, DURABLE_DIRTY_USAGE_REPLAY_OBJECT, bytes).await { + Ok(()) => true, + Err(err) => { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "write_failed", + error = ?err, + "Durable scanner dirty usage journal write failed" + ); + false + } + } +} + +async fn delete_durable_dirty_usage_journal(store: Arc) -> bool { + match ecstore_config::com::delete_config(store, DURABLE_DIRTY_USAGE_REPLAY_OBJECT).await { + Ok(()) | Err(Error::ConfigNotFound) => true, + Err(err) => { + warn!( + event = EVENT_SCANNER_DIRTY_USAGE_JOURNAL, + component = LOG_COMPONENT_STORAGE, + subsystem = LOG_SUBSYSTEM_SCANNER, + state = "delete_failed", + error = ?err, + "Durable scanner dirty usage journal delete failed" + ); + false + } + } +} + +fn durable_dirty_usage_journal_scope(object: &str) -> JournalScope { + match durable_dirty_usage_top_level_entry(object) { + Some(entry) => JournalScope::TopLevelEntries(BTreeSet::from([entry])), + None => JournalScope::WholeBucket, + } +} + +fn merge_journal_scope(current: &mut JournalScope, incoming: JournalScope) { + let overflowed = match (&mut *current, incoming) { + (JournalScope::WholeBucket, _) | (_, JournalScope::WholeBucket) => { + *current = JournalScope::WholeBucket; + false + } + (JournalScope::TopLevelEntries(entries), JournalScope::TopLevelEntries(incoming)) => { + entries.extend(incoming); + entries.len() > DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES + } + }; + if overflowed { + *current = JournalScope::WholeBucket; + } +} + +fn durable_dirty_usage_producer_is_supported(producer: SegmentInvalidationProducerIdentity) -> bool { + matches!( + producer, + SegmentInvalidationProducerIdentity::PutObject + | SegmentInvalidationProducerIdentity::DeleteObject + | SegmentInvalidationProducerIdentity::DeleteMarker + | SegmentInvalidationProducerIdentity::CompleteMultipartUpload + | SegmentInvalidationProducerIdentity::AbortMultipartUpload + | SegmentInvalidationProducerIdentity::ObjectMetadata + | SegmentInvalidationProducerIdentity::BucketMetadata + | SegmentInvalidationProducerIdentity::Replication + | SegmentInvalidationProducerIdentity::TierTransition + | SegmentInvalidationProducerIdentity::TierExpiration + | SegmentInvalidationProducerIdentity::DirectoryObject + ) +} + +fn durable_dirty_usage_top_level_entry(object: &str) -> Option { + let (top_level_entry, _) = object.split_once('/').unwrap_or((object, "")); + (!top_level_entry.is_empty() + && top_level_entry != "." + && top_level_entry != ".." + && !object.starts_with('/') + && !top_level_entry.contains(['\\', '\0'])) + .then(|| top_level_entry.to_string()) +} + +#[cfg(test)] +mod tests { + use super::{DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES, JournalState}; + use rustfs_scanner::{ + ScannerDirtyUsageBucket, ScannerDurableDirtyUsageReplayEntry, ScannerDurableDirtyUsageReplayRecord, + ScannerDurableDirtyUsageReplayScope, SegmentInvalidationProducerIdentity, + }; + use std::collections::BTreeSet; + + #[test] + fn journal_state_merges_scopes_and_clears_only_confirmed_generations() { + let mut state = JournalState::default(); + state.record_mutation("photos".to_string(), "2026/object-a", 7, SegmentInvalidationProducerIdentity::Replication); + state.record_mutation( + "photos".to_string(), + "archive/object-b", + 9, + SegmentInvalidationProducerIdentity::TierExpiration, + ); + + let entries = state.replay_entries().expect("journal should encode a bounded bucket"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].generation, 9); + assert_eq!( + entries[0].producers, + BTreeSet::from([ + SegmentInvalidationProducerIdentity::Replication, + SegmentInvalidationProducerIdentity::TierExpiration + ]) + ); + assert_eq!( + entries[0].scope, + ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: BTreeSet::from(["2026".to_string(), "archive".to_string()]) + } + ); + + state.clear_buckets(&[ScannerDirtyUsageBucket { + bucket: "photos".to_string(), + generation: 7, + }]); + assert!(state.buckets.contains_key("photos")); + state.clear_buckets(&[ScannerDirtyUsageBucket { + bucket: "photos".to_string(), + generation: 9, + }]); + assert!(state.buckets.is_empty()); + } + + #[test] + fn journal_state_expands_to_whole_bucket_for_ambiguous_or_overflowing_scopes() { + let mut ambiguous = JournalState::default(); + ambiguous.record_mutation("photos".to_string(), "../bad", 7, SegmentInvalidationProducerIdentity::Replication); + assert_eq!( + ambiguous.replay_entries().expect("ambiguous object should still encode")[0].scope, + ScannerDurableDirtyUsageReplayScope::WholeBucket + ); + + let mut overflow = JournalState::default(); + for index in 0..=DURABLE_DIRTY_USAGE_REPLAY_MAX_TOP_LEVEL_ENTRIES { + overflow.record_mutation( + "photos".to_string(), + &format!("prefix-{index}/object"), + u64::try_from(index + 1).expect("test index fits in u64"), + SegmentInvalidationProducerIdentity::Replication, + ); + } + assert_eq!( + overflow.replay_entries().expect("overflow should encode as whole bucket")[0].scope, + ScannerDurableDirtyUsageReplayScope::WholeBucket + ); + } + + #[test] + fn journal_state_hydrates_replayed_records_before_later_flushes() { + let record = ScannerDurableDirtyUsageReplayRecord { + schema: 1, + cache_key_format: 1, + writer_epoch: "writer".to_string(), + entries: vec![ScannerDurableDirtyUsageReplayEntry { + bucket: "photos".to_string(), + generation: 7, + scope: ScannerDurableDirtyUsageReplayScope::TopLevelEntries { + entries: BTreeSet::from(["2026".to_string()]), + }, + producers: BTreeSet::from([SegmentInvalidationProducerIdentity::Replication]), + }], + }; + let mut state = JournalState::from_replay_record(record).expect("valid replay record should hydrate"); + state.record_mutation( + "videos".to_string(), + "clips/object", + 8, + SegmentInvalidationProducerIdentity::TierExpiration, + ); + + let entries = state.replay_entries().expect("hydrated state should remain encodable"); + assert_eq!(entries.len(), 2); + assert!(entries.iter().any(|entry| entry.bucket == "photos" && entry.generation == 7)); + assert!(entries.iter().any(|entry| entry.bucket == "videos" && entry.generation == 8)); + } + + #[test] + fn journal_state_hydration_preserves_replayed_bucket_scope_and_producers() { + let record = ScannerDurableDirtyUsageReplayRecord { + schema: 1, + cache_key_format: 1, + writer_epoch: "writer".to_string(), + entries: vec![ScannerDurableDirtyUsageReplayEntry { + bucket: "photos".to_string(), + generation: 7, + scope: ScannerDurableDirtyUsageReplayScope::WholeBucket, + producers: BTreeSet::from([SegmentInvalidationProducerIdentity::Replication]), + }], + }; + let mut state = JournalState::from_replay_record(record).expect("valid replay record should hydrate"); + state.record_mutation( + "photos".to_string(), + "2026/object", + 8, + SegmentInvalidationProducerIdentity::TierExpiration, + ); + + let entries = state.replay_entries().expect("hydrated state should remain encodable"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].bucket, "photos"); + assert_eq!(entries[0].generation, 8); + assert_eq!(entries[0].scope, ScannerDurableDirtyUsageReplayScope::WholeBucket); + assert_eq!( + entries[0].producers, + BTreeSet::from([ + SegmentInvalidationProducerIdentity::Replication, + SegmentInvalidationProducerIdentity::TierExpiration + ]) + ); + } + + #[test] + fn journal_state_drops_invalid_generation_or_non_production_identity_fail_closed() { + let mut state = JournalState::default(); + state.record_mutation("photos".to_string(), "2026/object", 7, SegmentInvalidationProducerIdentity::Replication); + state.record_mutation( + "videos".to_string(), + "clip/object", + u64::MAX, + SegmentInvalidationProducerIdentity::Replication, + ); + assert!(state.buckets.is_empty()); + + state.record_mutation("photos".to_string(), "2026/object", 7, SegmentInvalidationProducerIdentity::Unknown); + assert!(state.buckets.is_empty()); + } +} diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index e05c4f4e8..c47e267f1 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -74,6 +74,7 @@ use super::storage_api::ecstore_object::{ EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, }; +use crate::runtime_sources::current_kms_runtime_service_manager; use crate::storage::access::{ReqInfo, request_context_from_req, resource_free_condition_values}; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; #[cfg(feature = "rio-v2")] @@ -2758,7 +2759,32 @@ async fn apply_managed_encryption_material_inner( // key it will actually be encrypted under. authorize_sse_kms_key(principal, encryption_type, KmsAction::GenerateDataKeyAction, &kms_key_to_use).await?; - let provider = get_sse_dek_provider().await?; + // A node-local master key is the explicit SSE-S3 fallback. Letting it serve + // an SSE-KMS request would persist an `aws:kms` marker and a KMS key id + // that never wrapped the data key, so refuse rather than downgrade. + // + // The refusal sits after the authorization gate so an unauthorized caller + // still sees AccessDenied whatever the KMS runtime state is, and it asks + // the resolved provider rather than a parallel availability signal, + // because the provider is what actually wraps the DEK. + let provider = match get_sse_dek_provider().await { + Ok(provider) => { + if matches!(encryption_type, SSEType::SseKms) && provider.wraps_dek_with_local_master_key() { + return Err(sse_kms_unavailable_error(kms_configured_but_unavailable().await)); + } + provider + } + // With no master key set the local fallback fails with an SSE-S3-worded + // configuration error. An SSE-KMS request never asked for that provider, + // so it gets the SSE-KMS refusal instead of a message naming the wrong + // scheme. + Err(err) => { + if matches!(encryption_type, SSEType::SseKms) && runtime_sources::current_encryption_service().await.is_none() { + return Err(sse_kms_unavailable_error(kms_configured_but_unavailable().await)); + } + return Err(err); + } + }; let object_context = build_object_encryption_context(bucket, key, ssekms_context.as_ref()); let (data_key, encrypted_data_key) = provider.generate_sse_dek(&object_context, &kms_key_to_use).await?; @@ -2790,6 +2816,27 @@ async fn apply_managed_encryption_material_inner( }) } +/// Whether this node has a KMS configured that is not currently serving, which +/// separates a transient outage (retryable, 503) from a deployment that never +/// configured KMS at all (a client-side configuration error, 400). +async fn kms_configured_but_unavailable() -> bool { + match current_kms_runtime_service_manager() { + Some(manager) => !matches!(manager.get_status().await, rustfs_kms::KmsServiceStatus::NotConfigured), + None => false, + } +} + +fn sse_kms_unavailable_error(configured_but_unavailable: bool) -> ApiError { + if configured_but_unavailable { + return ApiError::from(StorageError::other(KmsUnavailableError)); + } + ApiError { + code: S3ErrorCode::InvalidRequest, + message: "SSE-KMS requires a configured and running KMS service".to_string(), + source: None, + } +} + async fn apply_managed_decryption_material( bucket: &str, key: &str, @@ -3236,6 +3283,16 @@ pub trait SseDekProvider: Send + Sync { ))) } + /// Whether this provider wraps data keys with a node-local master key + /// instead of a KMS service. + /// + /// SSE-KMS must never be served by such a provider: the stored object would + /// claim `aws:kms` and name a KMS key id that never wrapped anything. + /// Defaults to false so only the local fallback has to declare itself. + fn wraps_dek_with_local_master_key(&self) -> bool { + false + } + /// Decrypt a DEK from positively identified legacy managed metadata. #[cfg(feature = "rio-v2")] async fn decrypt_legacy_sse_dek( @@ -3758,6 +3815,10 @@ impl LocalSseDekProvider { #[async_trait] impl SseDekProvider for LocalSseDekProvider { + fn wraps_dek_with_local_master_key(&self) -> bool { + true + } + async fn generate_sse_dek( &self, _context: &ObjectEncryptionContext, @@ -4344,6 +4405,127 @@ mod tests { assert_eq!(super::kms_data_plane_error_class(&missing), "key_not_found"); } + #[test] + fn sse_kms_never_falls_back_to_the_local_sse_s3_provider() { + let unconfigured = super::sse_kms_unavailable_error(false); + assert_eq!(unconfigured.code, S3ErrorCode::InvalidRequest); + assert!(unconfigured.message.contains("SSE-KMS requires")); + + let stopped = super::sse_kms_unavailable_error(true); + assert_eq!(stopped.code, S3ErrorCode::ServiceUnavailable); + } + + fn managed_write(algorithm: &'static str, kms_key_id: Option<&str>) -> EncryptionRequest<'static> { + EncryptionRequest { + bucket: "finance", + key: "ledger.csv", + server_side_encryption: Some(ServerSideEncryption::from_static(algorithm)), + ssekms_key_id: kms_key_id.map(str::to_string), + ssekms_context: None, + sse_customer_algorithm: None, + sse_customer_key: None, + sse_customer_key_md5: None, + content_size: 128, + principal: None, + } + } + + /// A bucket default naming a KMS key, on a node with no KMS, used to write + /// the object under the local master key while stamping `aws:kms` and that + /// never-consulted key id into the metadata (backlog#2368 B4). + #[tokio::test] + async fn sse_kms_write_is_refused_when_only_a_local_master_key_is_available() { + let _guard = lock_sse_test_state().await; + reset_sse_dek_provider(); + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", None::), + ("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode_to_string([9u8; 32]))), + ], + async { + let error = sse_encryption(managed_write( + ServerSideEncryption::AWS_KMS, + Some("arn:aws:kms:us-east-1:123:key/nonexistent"), + )) + .await + .expect_err("SSE-KMS must not be served by the local master key"); + + assert_eq!(error.code, S3ErrorCode::InvalidRequest); + assert!(error.message.contains("SSE-KMS requires"), "message was {}", error.message); + }, + ) + .await; + + reset_sse_dek_provider(); + } + + /// Without a master key the local fallback fails with an SSE-S3-worded + /// configuration error. An SSE-KMS request must not be told to set + /// `RUSTFS_SSE_S3_MASTER_KEY`. + #[tokio::test] + async fn sse_kms_refusal_names_sse_kms_rather_than_the_sse_s3_master_key() { + let _guard = lock_sse_test_state().await; + reset_sse_dek_provider(); + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", None::), + ("RUSTFS_SSE_S3_MASTER_KEY", None::), + ], + async { + let error = sse_encryption(managed_write(ServerSideEncryption::AWS_KMS, Some("finance-key"))) + .await + .expect_err("SSE-KMS must be refused when no KMS is configured"); + + assert_eq!(error.code, S3ErrorCode::InvalidRequest); + assert!(error.message.contains("SSE-KMS requires"), "message was {}", error.message); + assert!( + !error.message.contains("RUSTFS_SSE_S3_MASTER_KEY"), + "an SSE-KMS refusal must not name the SSE-S3 master key: {}", + error.message + ); + }, + ) + .await; + + reset_sse_dek_provider(); + } + + /// The SSE-S3 local fallback itself is unchanged: refusing SSE-KMS must not + /// take the documented no-KMS deployment down with it. + #[tokio::test] + async fn sse_s3_write_still_uses_the_local_master_key_fallback() { + let _guard = lock_sse_test_state().await; + reset_sse_dek_provider(); + + async_with_vars( + [ + ("__RUSTFS_SSE_SIMPLE_CMK", None::), + ("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode_to_string([9u8; 32]))), + ], + async { + let material = sse_encryption(managed_write(ServerSideEncryption::AES256, None)) + .await + .expect("SSE-S3 keeps its local master key fallback") + .expect("managed sse-s3 material"); + + assert_eq!(material.sse_type, SSEType::SseS3); + assert_eq!(material.algorithm, ServerSideEncryption::AES256); + + // No object may claim aws:kms while its DEK is wrapped locally. + let metadata = encryption_material_to_metadata(&material).expect("sse-s3 metadata should serialize"); + assert!( + !metadata.iter().any(|(_, value)| value == ServerSideEncryption::AWS_KMS), + "local-master-key material must never be stamped aws:kms: {metadata:?}" + ); + }, + ) + .await; + + reset_sse_dek_provider(); + } + #[test] fn parse_simple_sse_cmk_accepts_valid_32_byte_key() { let mut key = [0u8; 32]; @@ -5757,8 +5939,11 @@ mod tests { }) .await .expect_err("mismatched kms context should fail"); - assert_eq!(err.code, S3ErrorCode::InternalError); - assert_eq!(err.message, ApiError::error_code_to_message(&S3ErrorCode::InternalError)); + assert_eq!(err.code, S3ErrorCode::InvalidRequest); + assert_eq!( + err.message, + "Encryption context mismatch: Context mismatch for key 'tenant': expected 'alpha', got 'beta'" + ); assert_eq!(super::kms_data_plane_error_class(&err), "context_mismatch"); manager.stop().await.expect("kms service should stop cleanly"); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 4dbc1b341..5c8a095ec 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -475,6 +475,8 @@ pub(crate) mod ecstore_disk { RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout, validate_batch_read_version_item_count, }; + #[cfg(test)] + pub(crate) use rustfs_ecstore::api::disk::{DiskOption, new_disk}; pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce}; } @@ -952,6 +954,25 @@ pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperti } pub(crate) async fn init_background_replication(store: Arc) { + let durable_dirty_usage_journal = super::scanner_dirty_journal::start_durable_dirty_usage_journal(store.clone()).await; + let mutation_journal = durable_dirty_usage_journal.clone(); + rustfs_scanner::set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, producer| { + mutation_journal.record_committed_mutation(bucket, object, producer); + }))); + ecstore_bucket::replication::set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| { + let producer = match source { + ecstore_bucket::replication::ScannerDirtyUsageMutationSource::Replication => { + rustfs_scanner::SegmentInvalidationProducerIdentity::Replication + } + ecstore_bucket::replication::ScannerDirtyUsageMutationSource::TierExpiration => { + rustfs_scanner::SegmentInvalidationProducerIdentity::TierExpiration + } + }; + rustfs_scanner::record_dirty_usage_object_from_producer(bucket, object, producer); + }))); + rustfs_scanner::set_scanner_dirty_usage_clear_observer(Some(Arc::new(move |cleared| { + durable_dirty_usage_journal.clear_confirmed_buckets(cleared); + }))); ecstore_bucket::replication::init_background_replication(store).await; } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 9aea57c03..088e61b03 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -211,12 +211,19 @@ pub(crate) mod server { pub(crate) mod readiness { pub(crate) mod contract { pub(crate) mod admin { - pub(crate) use super::super::super::super::storage_contracts::StorageAdminApi; + pub(crate) use super::super::super::super::storage_contracts::{DiskSetSelector, StorageAdminApi}; } } + pub(crate) use crate::storage::storage_api::ecstore_disk::DiskStore; pub(crate) use crate::storage::storage_api::{Endpoint, EndpointServerPools, is_dist_erasure}; + pub(crate) fn disk_endpoint_snapshot(disk: &DiskStore) -> Endpoint { + crate::storage::storage_api::ecstore_disk::DiskAPI::endpoint(disk.as_ref()) + } + + #[cfg(test)] + pub(crate) use crate::storage::storage_api::ecstore_disk::{DiskOption, new_disk}; #[cfg(test)] pub(crate) use crate::storage::storage_api::{Endpoints, PoolEndpoints}; } diff --git a/scripts/README.md b/scripts/README.md index 50f8ea7a8..b661a6a3f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -55,8 +55,34 @@ their issue closes. | `run.ps1` | dev-tool | Windows counterpart of `run.sh` | — | | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` | | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` | +| `check_test_wiring.py` | dev-tool | Validates test wiring and assembles one or more measured Scanner/Heal release descriptors into a single release evidence bundle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --self-test` | | `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` | +| `run_scanner_heal_authority_evidence.py` | dev-tool | Assembles measured Scanner/Heal G01 root/quota authority release descriptors from operator-collected authority artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_authority_evidence.sh` | +| `run_scanner_heal_checkpoint_crash_evidence.py` | dev-tool | Assembles measured Scanner/Heal G02/R-E checkpoint and restart release descriptors from scanner restart diagnostic reports | `diagnose_scanner_enumeration_restart.py`; `test_scanner_heal_checkpoint_crash_evidence.sh` | +| `run_scanner_heal_g14_multiset_evidence.py` | dev-tool | Assembles measured Scanner/Heal G14 same-window EC8+4 multi-set/multi-pool release descriptors from e2e case directories or an operator-collected proof | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_g14_multiset_evidence.sh` | +| `run_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Runs the G09 mixed-version and rollback upgrade E2E lanes against a pinned previous release and verifies the raw evidence artifacts | `docs/testing/ci-gates.md`; `.github/workflows/e2e-upgrade.yml`; `test_scanner_heal_g09_upgrade_evidence.sh` | +| `run_scanner_heal_linux_evidence_plan.py` | dev-tool | Writes the unified Scanner/Heal Linux release-evidence execution manifest and can run lightweight preflight checks without producing measured evidence | `docs/testing/ci-gates.md`; `test_scanner_heal_linux_evidence_plan.sh` | +| `run_scanner_heal_scoped_ack_evidence.py` | dev-tool | Assembles measured Scanner/Heal G03 scoped ACK publication and mixed-peer fallback release descriptors | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_scoped_ack_evidence.sh` | +| `run_scanner_heal_legacy_rollback_evidence.py` | dev-tool | Assembles measured Scanner/Heal R-L legacy source-conflict, migration-gap, and source-retirement release descriptors | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_legacy_rollback_evidence.sh` | +| `run_scanner_heal_mrf_evidence.py` | dev-tool | Assembles measured Scanner/Heal G07/G08/P4 MRF release descriptors from W13 raw artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_w13_mrf_evidence.sh` | +| `run_scanner_heal_scheduler_pressure_evidence.py` | dev-tool | Assembles measured Scanner/Heal G10/P1/P3 scheduler-pressure release descriptors from a completed measured ABBA run, recovery-window proof, and profile artifacts | `docs/operations/scanner-benchmark-runbook.md`; `test_scanner_heal_scheduler_pressure_evidence.sh` | +| `run_scanner_heal_status_outcome_probe.py` | dev-tool | Normalizes live Scanner/Heal status/outcome observations into the measured G05/G06/R-D raw artifacts consumed by the descriptor producer | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_status_outcome_evidence.sh` | +| `run_scanner_heal_status_outcome_evidence.py` | dev-tool | Assembles measured Scanner/Heal G05/G06/R-D status-and-outcome release descriptors from same-run status, compatibility, and disposition artifacts | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_status_outcome_evidence.sh` | +| `run_scanner_heal_maintenance_evidence.py` | dev-tool | Assembles measured Scanner/Heal G11/G13 maintenance-producer release descriptors from operator-collected proof JSON | `.config/scanner-heal-required-tests.json`; `test_scanner_heal_maintenance_evidence.sh` | +| `run_scanner_heal_w13_mrf_evidence.sh` | dev-tool | Runs the W13 durable MRF replay lanes and writes G07/G08/P4 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w13_mrf_evidence.sh` | +| `run_scanner_heal_w16_recovery_evidence.sh` | dev-tool | Runs the W16 recovery-intent and quota authority lanes and writes G04/G12 bundle-ready evidence descriptors | `docs/testing/ci-gates.md`; `test_scanner_heal_w16_recovery_evidence.sh` | | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — | +| `test_scanner_heal_checkpoint_crash_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal checkpoint/crash evidence assembler | — | +| `test_scanner_heal_authority_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G01 authority evidence assembler | — | +| `test_scanner_heal_scoped_ack_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G03 scoped ACK evidence assembler | — | +| `test_scanner_heal_legacy_rollback_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal R-L legacy rollback evidence assembler | — | +| `test_scanner_heal_g14_multiset_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G14 multi-set/multi-pool evidence assembler | — | +| `test_scanner_heal_scheduler_pressure_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal scheduler-pressure evidence assembler | — | +| `test_scanner_heal_status_outcome_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal status-and-outcome evidence assembler | — | +| `test_scanner_heal_maintenance_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G11/G13 maintenance evidence assembler | — | +| `test_scanner_heal_g09_upgrade_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal G09 upgrade evidence runner | — | +| `test_scanner_heal_w16_recovery_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal W16 recovery evidence runner | — | +| `test_scanner_heal_w13_mrf_evidence.sh` | dev-tool | Shell self-test for the Scanner/Heal W13 MRF evidence runner | — | | `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` | | `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` | | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) | diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index e56774130..840ad4aaf 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -7,13 +7,14 @@ import hashlib import json import os import re +import shutil import subprocess import sys import tempfile import unittest import uuid import xml.etree.ElementTree as ET -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from unittest import mock from pathlib import Path from zoneinfo import ZoneInfo, ZoneInfoNotFoundError @@ -23,10 +24,484 @@ try: except ModuleNotFoundError: import tomli as tomllib -from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json +from scanner_abba import ( + MAX_JSON_BYTES, + RELEASE_PROFILE_ARTIFACTS, + SCENARIOS, + digest, + number, + read_json, + require, + sha, + write_json, +) ROOT = Path(__file__).resolve().parents[1] +SCANNER_HEAL_REGISTRY_SCHEMA_MAX = 2 +SCANNER_HEAL_RELEASE_REQUIRED_GATES = ( + "G01", "G02", "G03", "G04", "G05", "G06", "G07", "G08", "G09", "G10", "G11", "G12", "G13", "G14", + "P1", "P2", "P3", "P4", "R-E", "R-D", "R-L", +) +SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS = { + "G01": ( + "root_authority_evidence", + "quota_authority_evidence", + ), + "G02": ( + "bounded_checkpoint_oracle", + "independent_version_inventory", + ), + "G03": ( + "durable_root_publication_proof", + "scoped_ack_request_identity", + "participating_peer_capability_snapshot", + "mixed_peer_ack_fallback_oracle", + ), + "G04": ( + "cache_boundary_crash_evidence", + "root_floor_intent_crash_evidence", + ), + "G05": ( + "per_object_outcome_oracle", + "terminal_retention_bounds", + ), + "G06": ( + "concurrent_status_evidence", + "legacy_client_compatibility", + "truncation_behavior", + ), + "G07": ( + "mrf_responsibility_oracle", + "commit_boundary_crash_matrix", + ), + "G08": ( + "mrf_capacity_evidence", + "disk_full_matrix", + "replica_loss_matrix", + ), + "G09": ( + "mixed_version_reader_evidence", + "mixed_version_writer_evidence", + "rollback_payload_evidence", + ), + "G10": ( + "scheduler_bound_evidence", + "pressure_recovery_evidence", + ), + "G11": ( + "maintenance_producer_matrix", + "complete_producer_inventory", + "segment_activation_preflight", + ), + "G12": ( + "reset_quota_path_evidence", + "settlement_quota_path_evidence", + ), + "G13": ( + "quorum_minus_one_matrix", + "unknown_disk_remount_matrix", + "object_lock_dry_run_grace_evidence", + ), + "G14": ( + "same_window_field_evidence", + "ec8_4_evidence", + "multi_set_evidence", + "multi_pool_evidence", + "distributed_segment_invalidation_evidence", + ), + "P1": ( + "cold_walk_share_measurement", + "foreground_latency_throughput_measurement", + "profile_evidence", + ), + "P2": ( + "post_stop_convergence_measurement", + "cold_segment_reuse_measurement", + ), + "P3": ( + "two_hour_pressure_measurement", + "heal_capacity_measurement", + "recovery_window_measurement", + ), + "P4": ( + "mrf_scale_measurement", + "mrf_replay_cost_measurement", + "retained_responsibility_evidence", + "mrf_cleanup_gc_soak_evidence", + ), + "R-E": ( + "fixed_budget_restart_evidence", + "enumeration_evidence", + "classification_evidence", + ), + "R-D": ( + "manager_disposition_evidence", + "event_disposition_evidence", + "ledger_disposition_evidence", + "grace_handling", + ), + "R-L": ( + "legacy_source_conflict_evidence", + "migration_gap_evidence", + "crash_safe_source_retirement_evidence", + ), +} +SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS = { + "G01": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G01"], + "G02": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G02"], + "G03": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G03"], + "G04": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G04"], + "G05": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G05"], + "G06": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G06"], + "G07": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G07"], + "G08": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G08"], + "G09": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G09"], + "G10": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G10"], + "G11": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G11"], + "G12": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G12"], + "G13": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G13"], + "G14": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["G14"], + "P1": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["P1"], + "P2": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["P2"], + "P3": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["P3"], + "P4": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["P4"], + "R-E": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["R-E"], + "R-D": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["R-D"], + "R-L": SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS["R-L"], +} +SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES = { + ("G03", "durable_root_publication_proof"): "durable-root-publication", + ("G03", "scoped_ack_request_identity"): "scoped-ack-request", + ("G03", "participating_peer_capability_snapshot"): "peer-capability-snapshot", + ("G03", "mixed_peer_ack_fallback_oracle"): "mixed-peer-ack-fallback", + ("G09", "mixed_version_reader_evidence"): "mixed-version-reader", + ("G09", "mixed_version_writer_evidence"): "mixed-version-writer", + ("G09", "rollback_payload_evidence"): "rollback-payload", + ("R-L", "legacy_source_conflict_evidence"): "legacy-source-conflict", + ("R-L", "migration_gap_evidence"): "migration-gap", + ("R-L", "crash_safe_source_retirement_evidence"): "crash-safe-source-retirement", +} +SCANNER_HEAL_RELEASE_G01_ROOT_AUTHORITY_CASES = ( + "root-cas-success", + "root-readback-success", + "incomplete-root-rejected", + "stale-root-rejected", +) +SCANNER_HEAL_RELEASE_G01_QUOTA_AUTHORITY_CASES = ( + "quota-floor-readback", + "quota-over-limit-rejected", + "rejected-object-invisible", + "quota-fails-closed-without-authority", +) +SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS = { + ("G07", "mrf_responsibility_oracle"), + ("G07", "commit_boundary_crash_matrix"), + ("P4", "mrf_replay_cost_measurement"), + ("P4", "retained_responsibility_evidence"), + ("P4", "mrf_cleanup_gc_soak_evidence"), +} +SCANNER_HEAL_RELEASE_MRF_ARTIFACT_KINDS = { + ("G07", "mrf_responsibility_oracle"): "mrf-durable-responsibility-oracle", + ("G07", "commit_boundary_crash_matrix"): "mrf-commit-boundary-crash-matrix", + ("G08", "mrf_capacity_evidence"): "mrf-capacity-boundary", + ("G08", "disk_full_matrix"): "mrf-disk-full-enospc-matrix", + ("G08", "replica_loss_matrix"): "mrf-replica-loss-matrix", + ("P4", "mrf_scale_measurement"): "mrf-scale-measurement", + ("P4", "mrf_replay_cost_measurement"): "mrf-replay-cost-measurement", + ("P4", "retained_responsibility_evidence"): "mrf-retained-responsibility-soak", + ("P4", "mrf_cleanup_gc_soak_evidence"): "mrf-cleanup-gc-soak", +} +SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS = ( + "missing_producer_identity", + "missing_durable_journal_replay", + "restart_gap", + "generation_gap", + "overflow", + "stale_ack_generation", + "missing_cold_zero_walk_oracle", + "distributed_without_peer_invalidation", +) +SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS = ( + "source", + "bucket_incarnation", + "key_format", + "baseline_scan_plan_digest", + "process_epoch", + "generation_window", + "producer_identities", + "durable_dirty_producer_journal", + "ack_generation_guard", +) +SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES = ( + "put_object", + "delete_object", + "delete_marker", + "complete_multipart_upload", + "abort_multipart_upload", + "object_metadata", + "bucket_metadata", + "replication", + "tier_transition", + "tier_expiration", + "directory_object", +) +SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES = ( + "put", + "delete", + "delete_marker", + "multipart", + "replication", + "tier", + "directory_object", +) +SCANNER_HEAL_DURABLE_JOURNAL_FAIL_CLOSED_CASES = ( + "corrupt-record", + "mixed-version-producer", + "oversized-record", + "unsupported-producer", + "invalid-generation", +) +SCANNER_HEAL_RELEASE_G11_REQUIRED_CASES = { + "maintenance_producer_matrix": ( + "object-mutation-producers", + "metadata-mutation-producers", + "replication-tier-producers", + "directory-object-producer", + ), + "complete_producer_inventory": SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES, +} +SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES = { + "quorum_minus_one_matrix": ( + "read-quorum-minus-one", + "write-quorum-minus-one", + "heal-quorum-minus-one", + "restart-quorum-minus-one", + ), + "unknown_disk_remount_matrix": ( + "unknown-disk-excluded-from-quorum", + "known-disk-remount-rejoins", + "stale-disk-remount-rejected", + ), + "object_lock_dry_run_grace_evidence": ( + "object-lock-delete-denied", + "dry-run-does-not-mutate", + "grace-window-retains-terminal-outcome", + ), +} +SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES = { + "mrf_responsibility_oracle": ( + "legacy-journal-replay", + "scoped-journal-replay", + "committed-checkpoint-replay", + ), + "commit_boundary_crash_matrix": ( + "before-committed-payload", + "after-payload-before-manifest", + "after-manifest-before-cleanup", + "restart-replay-before-successor", + ), +} +SCANNER_HEAL_RELEASE_G08_REQUIRED_CASES = { + "mrf_capacity_evidence": ( + "queue-count-limit", + "journal-byte-limit", + "committed-payload-byte-limit", + ), + "disk_full_matrix": ( + "payload-write-enospc", + "manifest-write-enospc", + "journal-write-enospc", + "cleanup-delete-enospc", + ), + "replica_loss_matrix": ( + "single-replica-loss", + "quorum-minus-one", + "all-replicas-unavailable", + ), +} +SCANNER_HEAL_RELEASE_MRF_CLEANUP_GC_SOAK_CASES = ( + "retained-anchor-survives-restart", + "verified-successor-allows-idle-gc", + "stale-legacy-journal-cleanup", + "repeated-replay-no-resurrection", +) +SCANNER_HEAL_RELEASE_CRASH_BOUNDARY_FIELDS = { + ("G04", "root_floor_intent_crash_evidence"): ( + "persist-failure-no-202", + "same-key-retry-reuses-intent", + "different-params-conflict", + "process-restart-replay", + ), +} +SCANNER_HEAL_RELEASE_G12_QUOTA_PATH_CASES = { + "reset_quota_path_evidence": ( + "storage-owner-reconstruction", + "future-reservation-protocol-fail-closed", + "reservation-ledger-retained", + ), + "settlement_quota_path_evidence": ( + "distributed-hard-quota-admission", + "quota-stats-current-usage-observed", + "oversized-put-rejected", + "rejected-object-not-visible", + ), +} +SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES = ( + "object-repaired", + "object-already-healthy", + "object-skipped-by-policy", + "object-failed-and-retained", +) +SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES = ( + "finished-retained-until-window", + "failed-retained-until-window", + "canceled-retained-until-window", + "expired-terminal-pruned-after-window", +) +SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES = ( + "status-during-admin-heal", + "status-during-background-heal", + "status-while-peer-down", + "status-after-peer-rejoin", +) +SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES = ( + "rustfs-admin-v3-background-heal-status", + "minio-admin-v3-background-heal-status", + "heal-client-token-empty-body", + "node-heal-status-v1-wire", +) +SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES = ( + "oversize-node-status-reject", + "truncated-node-status-reject", + "trailing-data-node-status-reject", +) +SCANNER_HEAL_RELEASE_RD_MANAGER_CASES = ( + "accepted", + "coalesced-duplicate", + "rejected-policy", + "terminal-retained", +) +SCANNER_HEAL_RELEASE_RD_EVENT_CASES = ( + "event-repaired", + "event-failed", + "event-skipped", + "event-grace-retained", +) +SCANNER_HEAL_RELEASE_RD_LEDGER_CASES = ( + "ledger-recorded", + "ledger-replayed", + "ledger-discharged", + "ledger-pruned-after-grace", +) +SCANNER_HEAL_RELEASE_RD_GRACE_CASES = ( + "grace-open-retains-disposition", + "grace-expired-prunes-terminal", + "restart-preserves-grace-clock", +) +SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES = { + "legacy_source_conflict_evidence": ( + "data-movement-owned-target-equivalence", + "newer-target-conflict-rejected", + "legacy-part-checksum-gap-rejected", + ), + "migration_gap_evidence": ( + "encrypted-iam-decrypt-before-normalize", + "empty-legacy-source-inherits-prior-responsibilities", + "missing-corrupt-empty-sources-fail-closed", + ), + "crash_safe_source_retirement_evidence": ( + "source-change-capacity-failure-keeps-old-commit", + "torn-inactive-payload-keeps-previous-anchor", + "commit-boundary-lost-response-idempotent", + "successor-retry-validates-orphan", + ), +} +SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES = { + "durable_root_publication_proof": ( + "root-cas-success", + "root-readback-success", + "dirty-fallback-on-missing-proof", + ), + "scoped_ack_request_identity": ( + "bucket-incarnation", + "exact-generation", + "scanner-instance", + "participating-peer-set", + "cleared-count-bound", + ), + "participating_peer_capability_snapshot": ( + "supports-scoped-ack", + "probe-only-fallback", + "missing-capability-dirty", + ), + "mixed_peer_ack_fallback_oracle": ( + "legacy-peer-fallback", + "truncated-token-reject", + "restarted-peer-reject", + ), +} +SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS = { + "durable_root_publication_proof": ( + "root_cas_observed", + "root_readback_observed", + "dirty_fallback_on_missing_proof_observed", + ), + "scoped_ack_request_identity": ( + "bucket_incarnation_observed", + "exact_generation_observed", + "scanner_instance_observed", + "participating_peer_set_observed", + "whole_cycle_fallback_observed", + "cleared_count_bound_observed", + ), + "participating_peer_capability_snapshot": ( + "capability_probe_observed", + "scoped_ack_capability_observed", + "probe_only_fallback_observed", + "missing_capability_kept_dirty", + ), + "mixed_peer_ack_fallback_oracle": ( + "legacy_peer_fallback_observed", + "truncated_token_rejected", + "restarted_peer_rejected", + ), +} +SCANNER_HEAL_RELEASE_MIXED_VERSION_CASES = { + "mixed_version_reader_evidence": ( + "old-writer-new-reader", + "new-writer-old-reader", + ), + "mixed_version_writer_evidence": ( + "old-reader-new-writer", + "new-reader-old-writer", + ), + "rollback_payload_evidence": ( + "rollback-to-old", + "rollback-to-new", + "unknown-field-retained", + ), +} +SCANNER_HEAL_RELEASE_P4_RETAINED_RESPONSIBILITY_CASES = ( + "retain-pending-replay-anchor", + "verified-proof-discharges-anchor", + "idle-cleanup-reclaims-runtime-checkpoint", + "idle-cleanup-reclaims-replay-source", +) +SCANNER_HEAL_RELEASE_SCHEDULER_BOUNDS = ( + "admission-retry-idempotency", + "deadline-budget", + "lock-hold-bound", + "minimum-progress", +) +SCANNER_HEAL_RELEASE_PRESSURE_METRICS = ( + "foreground_p95_ms", + "foreground_p99_ms", + "throughput_ops", + "error_rate", + "heal_lock_wait_p99_ms", + "attempt_cost_samples", +) SCHEDULED_ALERT_WORKFLOWS = tuple( item["workflow"] for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text()) @@ -886,14 +1361,57 @@ def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> in return value +def evidence_string(value: object, name: str, pattern: str | None = None) -> str: + require(isinstance(value, str) and value.strip(), f"invalid string {name}") + if pattern is not None: + require(re.fullmatch(pattern, value) is not None, f"invalid string {name}") + return value + + +def evidence_timestamp(value: object, name: str) -> datetime: + text = evidence_string(value, name) + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError(f"invalid timestamp {name}") from error + require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed + + +def evidence_string_list(value: object, name: str) -> list[str]: + require( + isinstance(value, list) and value and all(isinstance(item, str) and item.strip() for item in value), + f"{name} must be a non-empty string list", + ) + return value + + +def evidence_exact_strings(value: object, expected: tuple[str, ...], name: str) -> list[str]: + strings = evidence_string_list(value, name) + observed = set(strings) + require(len(observed) == len(strings), f"duplicate {name}") + missing = sorted(set(expected) - observed) + require(not missing, f"missing {name}: {', '.join(missing)}") + unknown = sorted(observed - set(expected)) + require(not unknown, f"unknown {name}: {', '.join(unknown)}") + return strings + + +def scanner_heal_registry_schema(registry: dict[str, object]) -> int: + return evidence_integer(registry.get("schema"), "registry schema", 1, SCANNER_HEAL_REGISTRY_SCHEMA_MAX) + + def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]: registry = read_json(root / ".config/scanner-heal-required-tests.json") - evidence_integer(registry.get("schema"), "registry schema", 1, 1) + scanner_heal_registry_schema(registry) cases = registry.get("cases") require(isinstance(cases, dict) and cases, "invalid scanner/heal registry") names = set() for case_id, requirement in cases.items(): require(isinstance(case_id, str) and case_id, "invalid scanner/heal case identity") + lane = requirement.get("lane") + require(isinstance(lane, str) and re.fullmatch(r"[a-z0-9-]+", lane) is not None, + f"invalid nextest profile lane for {case_id}") oracle = requirement.get("oracle") require(isinstance(oracle, str) and oracle.endswith(".json"), f"invalid oracle for {case_id}") path = Path(oracle) @@ -902,10 +1420,113 @@ def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]: f"invalid evidence for {case_id}") require(type(requirement.get("unclean_shutdown_marker")) is bool, f"invalid unclean-shutdown marker expectation for {case_id}") + if "erasure" in requirement: + erasure = requirement["erasure"] + require(isinstance(erasure, dict), f"invalid erasure expectation for {case_id}") + data_blocks = evidence_integer(erasure.get("data_blocks"), f"{case_id} data_blocks", 1, 16) + parity_blocks = evidence_integer(erasure.get("parity_blocks"), f"{case_id} parity_blocks", 1, 16) + require(data_blocks >= parity_blocks, f"invalid erasure geometry for {case_id}") + expected_set_drives = requirement.get( + "erasure_set_drive_count", + requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"], + ) + require(data_blocks + parity_blocks == expected_set_drives, + f"erasure geometry differs from topology for {case_id}") + if "erasure_set_drive_count" in requirement: + erasure_set_drive_count = evidence_integer(requirement.get("erasure_set_drive_count"), + f"{case_id} erasure_set_drive_count", 1, 64) + total_drives = requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"] + require(total_drives % erasure_set_drive_count == 0, + f"erasure set drive count does not divide topology for {case_id}") + if "sets" in requirement: + require(evidence_integer(requirement.get("sets"), f"{case_id} sets", 1, 1024) + == total_drives // erasure_set_drive_count, f"set count differs from topology for {case_id}") + if "pools" in requirement: + evidence_integer(requirement.get("pools"), f"{case_id} pools", 1, 1024) names.add(oracle) return tuple(sorted(names)) +def scanner_heal_release_requirements(registry: dict[str, object]) -> tuple[dict[str, dict[str, object]], bool, list[str]]: + schema = scanner_heal_registry_schema(registry) + if schema == 1: + pending = registry.get("release_pending") + require(isinstance(pending, dict), "invalid scanner/heal release requirements") + requirements = {} + for gate, reason in pending.items(): + require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None, + "invalid scanner/heal release gate") + require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}") + requirements[gate] = {"gate": gate, "status": "pending", "lane": "schema-1-pending", + "description": reason, "requires": [reason]} + return requirements, False, ["schema-1-pending"] if requirements else [] + + lanes = registry.get("release_lanes") + cases = registry.get("cases") + require(isinstance(cases, dict) and cases, "invalid scanner/heal registry") + require(isinstance(lanes, dict) and lanes, "invalid scanner/heal release lanes") + lane_statuses = {} + lane_gates = {} + for lane_id, lane in lanes.items(): + require(isinstance(lane_id, str) and re.fullmatch(r"[a-z0-9-]+", lane_id) is not None, + "invalid scanner/heal release lane") + require(isinstance(lane, dict), f"invalid release lane {lane_id}") + status = lane.get("status") + require(status in ("implemented", "pending"), f"invalid release lane status for {lane_id}") + lane_statuses[lane_id] = status + if status == "implemented": + lane_cases = lane.get("cases") + require(isinstance(lane_cases, list) and lane_cases and all(isinstance(case, str) and case for case in lane_cases), + f"implemented release lane {lane_id} has no cases") + require(all(case in cases for case in lane_cases), f"implemented release lane {lane_id} has unknown cases") + else: + gates = lane.get("gates") + require(isinstance(gates, list) and gates and all(isinstance(gate, str) and gate for gate in gates), + f"pending release lane {lane_id} has no gates") + lane_gates[lane_id] = set(gates) + + raw_requirements = registry.get("release_requirements") + require(isinstance(raw_requirements, list) and raw_requirements, "invalid scanner/heal release requirements") + requirements: dict[str, dict[str, object]] = {} + for item in raw_requirements: + require(isinstance(item, dict), "invalid scanner/heal release requirement") + gate = item.get("gate") + require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None, + "invalid scanner/heal release gate") + require(gate not in requirements, f"duplicate scanner/heal release gate {gate}") + status = item.get("status") + require(status == "pending", f"release gate {gate} must stay pending until real evidence is registered") + lane = item.get("lane") + require(isinstance(lane, str) and lane in lane_statuses, f"unknown release lane for {gate}") + require(lane_statuses[lane] == "pending", f"pending release gate {gate} mapped to non-pending lane {lane}") + description = item.get("description") + require(isinstance(description, str) and description.strip(), f"missing release requirement for {gate}") + requires = item.get("requires") + require(isinstance(requires, list) and requires and + all(isinstance(requirement, str) and requirement.strip() for requirement in requires), + f"missing concrete evidence requirements for {gate}") + evidence_fields = item.get("evidence_fields", []) + require(isinstance(evidence_fields, list) and + all(isinstance(field, str) and re.fullmatch(r"[a-z0-9][a-z0-9_]*", field) is not None + for field in evidence_fields), + f"invalid evidence fields for release gate {gate}") + required_fields = set(SCANNER_HEAL_RELEASE_REQUIRED_EVIDENCE_FIELDS.get(gate, ())) + missing_fields = sorted(required_fields - set(evidence_fields)) + require(not missing_fields, + f"release gate {gate} missing required evidence fields: {', '.join(missing_fields)}") + requirements[gate] = item + + missing = sorted(set(SCANNER_HEAL_RELEASE_REQUIRED_GATES) - set(requirements)) + require(not missing, f"missing scanner/heal release requirements: {', '.join(missing)}") + for lane, gates in lane_gates.items(): + unknown = sorted(gates - set(requirements)) + require(not unknown, f"pending release lane {lane} has unknown gates: {', '.join(unknown)}") + mapped = {gate for gate, requirement in requirements.items() if requirement["lane"] == lane} + require(gates == mapped, f"pending release lane {lane} gates do not match release requirements") + pending_lanes = sorted(lane for lane, status in lane_statuses.items() if status == "pending") + return requirements, True, pending_lanes + + def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None: """Record an existing build; this command never builds or runs a test.""" require(not directory.exists(), "scanner/heal run directory must be new") @@ -969,7 +1590,7 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li """Validate one actual case, or fail the release while required lanes are pending.""" try: registry = read_json(root / ".config/scanner-heal-required-tests.json") - evidence_integer(registry.get("schema"), "registry schema", 1, 1) + scanner_heal_registry_schema(registry) require(registry.get("cases"), "invalid scanner/heal registry") selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]} run = read_json(directory / "run.json") @@ -1050,6 +1671,57 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch") for key in ("nodes", "drives_per_node"): evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16) + expected_set_drives = requirement.get( + "erasure_set_drive_count", + oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], + ) + require(oracle.get("erasure_set_drive_count") == expected_set_drives, + "oracle erasure set drive count mismatch") + if "sets" in requirement: + require(oracle.get("sets") == requirement["sets"], "oracle set count mismatch") + if "pools" in requirement: + require(oracle.get("pools") == requirement["pools"], "oracle pool count mismatch") + expected_outage_target_required = requirement.get("outage_target_manifest_required", True) + require(oracle.get("outage_target_manifest_required", True) is expected_outage_target_required, + "oracle outage target-manifest contract mismatch") + outage_write = oracle.get("outage_write") + outage_write_deferred = oracle.get("outage_write_deferred_until_rejoin", False) + require(type(outage_write_deferred) is bool, "invalid outage-write deferred flag") + if not expected_outage_target_required and outage_write is not None: + require(isinstance(outage_write, dict), "invalid optional outage-write diagnostic") + require(outage_write.get("attempted") is True, "optional outage-write diagnostic was not attempted") + require(outage_write.get("required") is False, "optional outage-write diagnostic required flag mismatch") + require(outage_write.get("accepted") is True, "optional outage-write final S3 body was not accepted") + outage_write_attempts = evidence_integer(outage_write.get("attempts"), "optional outage-write attempts", 1, 1024) + outage_write_service_unavailable = evidence_integer( + outage_write.get("service_unavailable"), "optional outage-write ServiceUnavailable count", 0, 1024 + ) + if outage_write_deferred: + require( + outage_write_service_unavailable == outage_write_attempts, + "deferred outage write must account for every down-window attempt", + ) + else: + require(outage_write_deferred is False, "outage-write deferred flag requires optional outage-write diagnostic") + if requirement.get("sets", 1) > 1 or requirement.get("pools", 1) > 1: + require(oracle.get("distributed_ec_invalidation") is True, + "oracle missing distributed EC invalidation proof") + evidence_integer(oracle.get("peer_count"), "oracle peer_count", 3, 64) + require(oracle.get("same_window_remote_proof") is True, "oracle missing same-window remote proof") + require(oracle.get("all_peers_bound_to_generation_window") is True, + "oracle missing peer generation-window binding") + expected_erasure = requirement.get("erasure") + if expected_erasure is not None: + require(isinstance(expected_erasure, dict), "invalid erasure expectation") + expected_data_blocks = evidence_integer(expected_erasure.get("data_blocks"), "expected EC data blocks", 1, 16) + expected_parity_blocks = evidence_integer(expected_erasure.get("parity_blocks"), "expected EC parity blocks", 1, 16) + require( + expected_data_blocks + expected_parity_blocks == expected_set_drives, + "expected EC geometry differs from topology", + ) + else: + expected_data_blocks = None + expected_parity_blocks = None evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1) evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1) require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed") @@ -1057,8 +1729,8 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li require(isinstance(objects, list) and requirement["min_objects"] <= len(objects) <= requirement["max_objects"], "incomplete/oversized object oracle") require(len({obj["key"] for obj in objects}) == len(objects), "duplicate object identity") - require(sum(obj["expected_physical"] is None for obj in objects) == 1, - "only the outage object may lack a pre-fault target manifest") + outage_object_count = sum(obj["expected_physical"] is None for obj in objects) + require(outage_object_count == 1, "only the outage object may lack a pre-fault target manifest") for obj in objects: require(isinstance(obj["key"], str) and 0 < len(obj["key"].encode()) <= 1024, "invalid object identity") require(obj["version_id"] is None, "this case only covers unversioned objects") @@ -1069,11 +1741,18 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li if obj["expected_physical"] is not None: require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest") for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []): + if not geometry["has_xl_meta"] and not expected_outage_target_required and obj["expected_physical"] is None: + continue data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16) parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16) - require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], - "EC geometry differs from this case's single set") + require(data + parity == expected_set_drives, + "EC geometry differs from this case's erasure set") + if expected_data_blocks is not None: + require(data == expected_data_blocks and parity == expected_parity_blocks, + "EC data/parity geometry differs from the required case") evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity) + if not expected_outage_target_required and obj["expected_physical"] is None and not physical["has_xl_meta"]: + continue require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata") parts = physical["expected_part_numbers"] require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage") @@ -1088,7 +1767,10 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings), "S3 listing differs from object oracle") if case_id == "release": - errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items()) + release_requirements, _, _ = scanner_heal_release_requirements(registry) + errors.extend( + f"pending {gate}: {requirement['description']}" for gate, requirement in release_requirements.items() + ) return errors except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: return [f"scanner/heal evidence rejected: {error}"] @@ -1097,15 +1779,10 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object]: """Return a compact release decision without weakening case validation.""" registry = read_json(root / ".config/scanner-heal-required-tests.json") - evidence_integer(registry.get("schema"), "registry schema", 1, 1) + scanner_heal_registry_schema(registry) cases = registry.get("cases") require(isinstance(cases, dict) and cases, "invalid scanner/heal registry") - pending = registry.get("release_pending") - require(isinstance(pending, dict), "invalid scanner/heal release requirements") - for gate, reason in pending.items(): - require(isinstance(gate, str) and re.fullmatch(r"[A-Z][A-Z0-9-]*", gate) is not None, - "invalid scanner/heal release gate") - require(isinstance(reason, str) and reason.strip(), f"missing release requirement for {gate}") + release_requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry) verified_cases = [] rejected_cases = [] @@ -1119,13 +1796,1654 @@ def scanner_heal_release_status(root: Path, directory: Path) -> dict[str, object "schema": 1, "decision": "blocked", "release_approved": False, - "release_schema_capable": False, + "release_schema_capable": release_schema_capable, "verified_cases": verified_cases, "rejected_cases": rejected_cases, - "pending_gates": sorted(pending), + "pending_gates": sorted(release_requirements), + "pending_lanes": pending_lanes, } +def release_bundle_artifact_path(bundle_path: Path, raw_path: object, gate: str, field: str) -> Path: + require(isinstance(raw_path, str) and raw_path.strip(), f"{gate}.{field} missing artifact") + path = Path(raw_path) + require(not path.is_absolute() and ".." not in path.parts, f"{gate}.{field} artifact path escapes bundle directory") + resolved = (bundle_path.parent / path).resolve() + require(resolved.is_relative_to(bundle_path.parent.resolve()), f"{gate}.{field} artifact path escapes bundle directory") + require(resolved.is_file(), f"{gate}.{field} artifact is missing") + require(resolved.stat().st_size > 0, f"{gate}.{field} artifact is empty") + return resolved + + +def is_json_artifact_format(value: str) -> bool: + normalized = value.lower() + return normalized == "json" or normalized.endswith("+json") + + +def release_bundle_json_artifact_mirrored_fields(gate: str, field: str) -> tuple[str, ...]: + fields: list[str] = [] + if gate == "G01": + if field == "root_authority_evidence": + fields.extend(( + "root_authority_cases", + "root_cas_observed", + "root_readback_observed", + "incomplete_root_rejected", + "stale_root_rejected", + )) + if field == "quota_authority_evidence": + fields.extend(( + "quota_authority_cases", + "quota_floor_readback_observed", + "over_limit_put_rejected", + "rejected_object_invisible", + "quota_fails_closed_without_authority", + )) + if gate in ("G03", "G09", "R-L"): + fields.extend(("versions", "mixed_version_role")) + if gate == "G02": + if field == "bounded_checkpoint_oracle": + fields.extend(( + "checkpoint_progress_bounded", + "raw_entry_budget", + "max_raw_entries_per_round", + "max_objects_processed_per_round", + "bounded_work_quantum_observed", + "durable_checkpoint_committed", + "no_unbounded_tail", + )) + if field == "independent_version_inventory": + fields.extend(( + "independent_version_inventory_observed", + "objects_expected", + "objects_retained", + "versions_retained", + "bytes_retained", + )) + if gate in ("G04", "G07", "R-E", "R-L"): + fields.append("crash_points") + if gate == "R-E": + if field == "fixed_budget_restart_evidence": + fields.extend(( + "fixed_budget_restart_converged", + "restart_rounds", + "raw_entry_budget", + "no_unbudgeted_final_sweep", + )) + if field == "enumeration_evidence": + fields.extend(( + "raw_enumeration_observed", + "durable_raw_page_commit_observed", + "raw_page_index_complete", + "enumeration_frontier_retained", + )) + if field == "classification_evidence": + fields.extend(( + "classification_observed", + "objects_processed", + "objects_retained", + "versions_retained", + "bytes_retained", + )) + if gate == "G03": + fields.append("scoped_ack_cases") + fields.extend(SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS[field]) + if gate == "G04" and field == "root_floor_intent_crash_evidence": + fields.extend(("durable_intent_cases", "persist_failure_blocks_acceptance")) + if gate == "G12": + fields.append("quota_path_cases") + if gate == "G05": + if field == "per_object_outcome_oracle": + fields.extend(("per_object_outcome_cases", "outcome_counts", "status_matches_object_oracle")) + if field == "terminal_retention_bounds": + fields.extend(( + "terminal_retention_cases", + "terminal_retention_window_seconds", + "max_terminal_record_age_seconds", + "terminal_records_pruned_after_window", + )) + if gate == "G06": + if field == "concurrent_status_evidence": + fields.extend(( + "concurrent_status_cases", + "status_samples", + "all_status_responses_http_success", + "partial_status_reports_degraded", + )) + if field == "legacy_client_compatibility": + fields.extend(( + "legacy_client_cases", + "rustfs_and_minio_paths_compatible", + "empty_body_status_requests_accepted", + )) + if field == "truncation_behavior": + fields.extend(("truncation_cases", "truncated_payloads_rejected", "max_status_payload_bytes")) + if gate == "G07": + fields.append({ + "mrf_responsibility_oracle": "mrf_responsibility_cases", + "commit_boundary_crash_matrix": "commit_crash_cases", + }[field]) + if gate == "G08": + fields.append({ + "mrf_capacity_evidence": "capacity_cases", + "disk_full_matrix": "disk_full_cases", + "replica_loss_matrix": "replica_loss_cases", + }[field]) + if gate == "G09": + fields.append("mixed_version_cases") + if field == "rollback_payload_evidence": + fields.append("rollback_payload_replayed") + if gate == "G11": + fields.extend({ + "maintenance_producer_matrix": ( + "producer_identities", + "producer_families", + "matrix_cases", + "durable_identity_observed", + "durable_journal_replay_observed", + "durable_journal_hydration_bound", + "generation_window_bound", + "restart_gap_absent", + "overflow_absent", + "ack_clear_generation_observed", + "invalid_journal_inputs_rejected", + ), + "complete_producer_inventory": ( + "required_producer_identities", + "observed_producer_identities", + "required_producer_families", + "observed_producer_families", + "missing_producer_identities", + "unknown_producer_excluded", + ), + "segment_activation_preflight": ( + "production_activation", + "scanner_segment_reuse_activated", + "proof_inputs", + "fail_closed_checks", + ), + }[field]) + if gate == "G13": + fields.extend({ + "quorum_minus_one_matrix": ( + "quorum_cases", + "no_success_at_quorum_minus_one", + "exact_quorum_restored", + ), + "unknown_disk_remount_matrix": ( + "remount_cases", + "unknown_disks_excluded", + "remounted_disks_revalidated", + "stale_incarnation_rejected", + ), + "object_lock_dry_run_grace_evidence": ( + "grace_cases", + "object_lock_denials_preserved", + "dry_run_mutation_count", + "grace_outcomes_retained", + ), + }[field]) + if gate == "G14": + fields.append("case_evidence") + if field == "same_window_field_evidence": + fields.append("same_window_fields") + if field == "ec8_4_evidence": + fields.append("topology") + if field == "multi_set_evidence": + fields.append("sets") + if field == "multi_pool_evidence": + fields.append("pools") + if field == "distributed_segment_invalidation_evidence": + fields.extend(( + "invalidation_domain", + "distributed_ec_invalidation", + "peer_count", + "same_window_remote_proof", + "all_peers_bound_to_generation_window", + )) + if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS: + fields.extend(("replayed_records", "responsibility_anchor_retained", "successor_snapshot_published")) + if gate == "P4" and field == "retained_responsibility_evidence": + fields.extend(( + "duration_seconds", + "retained_responsibility_cases", + "retention_window_seconds", + "idle_cleanup_observed", + "verified_proof_discharge_observed", + )) + if gate == "P4" and field == "mrf_cleanup_gc_soak_evidence": + fields.extend(( + "duration_seconds", + "cleanup_gc_cases", + "verified_idle_gc_observed", + "pending_responsibilities_after_gc", + "stale_journals_after_gc", + )) + if gate == "R-D": + if field == "manager_disposition_evidence": + fields.extend(("manager_disposition_cases", "manager_dispositions_are_terminal")) + if field == "event_disposition_evidence": + fields.extend(("event_disposition_cases", "events_correlate_to_manager_dispositions")) + if field == "ledger_disposition_evidence": + fields.extend(( + "ledger_disposition_cases", + "ledger_correlates_to_events", + "ledger_replay_preserves_terminal_disposition", + )) + if field == "grace_handling": + fields.extend(( + "grace_cases", + "grace_window_seconds", + "grace_retention_observed", + "grace_expiry_pruned_terminal_records", + )) + if gate == "R-L": + if field == "legacy_source_conflict_evidence": + fields.extend(( + "legacy_source_conflict_cases", + "source_conflicts_rejected", + "takeover_identity_bound", + "legacy_checksum_gap_rejected", + )) + if field == "migration_gap_evidence": + fields.extend(( + "migration_gap_cases", + "migration_gap_closed", + "legacy_sources_fail_closed", + "prior_responsibilities_inherited", + )) + if field == "crash_safe_source_retirement_evidence": + fields.extend(( + "source_retirement_cases", + "source_retirement_is_crash_safe", + "old_source_retained_until_successor", + "recovered_pending_migration", + )) + return tuple(dict.fromkeys(fields)) + + +PROFILE_ARTIFACT_REQUIRED_METRICS = { + "allocation-profile": ("resolved_samples", "allocation_bytes"), + "flamegraph": ("resolved_samples",), + "rss-samples": ("resolved_samples", "rss_peak_bytes"), + "save-frequency": ("resolved_samples", "save_operations", "saved_bytes"), +} + + +def validate_profile_json_artifact_payload(bundle_path: Path, path: Path, payload: dict[str, object], + gate: str, field: str, artifact_kind: str) -> None: + prefix = f"{gate}.{field}.{artifact_kind}" + raw_profile = release_bundle_artifact_path( + bundle_path, + payload.get("raw_profile_artifact"), + gate, + f"{field}.{artifact_kind}.raw_profile", + ) + require(sha(payload.get("raw_profile_sha256")) and digest(raw_profile) == payload["raw_profile_sha256"], + f"{prefix} raw profile hash mismatch") + raw_bytes = evidence_integer(payload.get("raw_profile_bytes"), f"{prefix}.raw_profile_bytes", 1, 2**63 - 1) + require(raw_bytes == raw_profile.stat().st_size, f"{prefix} raw profile bytes mismatch") + evidence_string(payload.get("raw_profile_format"), f"{prefix}.raw_profile_format", + r"[A-Za-z0-9][A-Za-z0-9._+:-]{1,63}") + require(raw_profile.resolve() != path.resolve(), f"{prefix} raw profile must be distinct from wrapper") + for metric in PROFILE_ARTIFACT_REQUIRED_METRICS[artifact_kind]: + evidence_integer(payload.get(metric), f"{prefix}.{metric}", 1, 2**63 - 1) + + +def validate_g14_json_artifact_payload(bundle_path: Path, payload: dict[str, object], + evidence: dict[str, object] | None, gate: str, field: str, + source_revision: str, window_id: str) -> None: + prefix = f"{gate}.{field}" + require(evidence is not None, f"{prefix} JSON artifact missing outer evidence context") + for mirror_field in release_bundle_json_artifact_mirrored_fields(gate, field): + require(payload.get(mirror_field) == evidence.get(mirror_field), + f"{prefix} JSON artifact {mirror_field} mismatch") + cases = payload.get("case_evidence") + require(isinstance(cases, list) and cases, f"{prefix} JSON artifact missing case_evidence") + seen_cases = set() + for index, item in enumerate(cases): + require(isinstance(item, dict), f"{prefix} JSON artifact case_evidence[{index}] must be an object") + case = evidence_string( + item.get("case"), + f"{prefix} JSON artifact case_evidence[{index}].case", + r"[A-Za-z0-9][A-Za-z0-9._:-]{1,127}", + ) + require(case not in seen_cases, f"{prefix} JSON artifact case_evidence has duplicate case: {case}") + seen_cases.add(case) + artifact = release_bundle_artifact_path( + bundle_path, + item.get("artifact"), + gate, + f"{field}.case_evidence.{case}", + ) + require(sha(item.get("sha256")) and digest(artifact) == item["sha256"], + f"{prefix} JSON artifact case_evidence[{index}] artifact hash mismatch") + require(item.get("source_revision") == source_revision, + f"{prefix} JSON artifact case_evidence[{index}].source_revision mismatch") + require(item.get("measurement_window_id") == window_id, + f"{prefix} JSON artifact case_evidence[{index}].measurement_window_id mismatch") + case_payload = read_json(artifact) + require(isinstance(case_payload, dict), f"{prefix} JSON artifact case_evidence[{index}] artifact must be a JSON object") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(case_payload.get(marker) is not True, + f"{prefix} JSON artifact case_evidence[{index}] artifact is {marker}") + require(case_payload.get("case") == case, + f"{prefix} JSON artifact case_evidence[{index}] artifact case mismatch") + if "source_revision" in case_payload: + require(case_payload["source_revision"] == source_revision, + f"{prefix} JSON artifact case_evidence[{index}] artifact source revision mismatch") + if "measurement_window_id" in case_payload: + require(case_payload["measurement_window_id"] == window_id, + f"{prefix} JSON artifact case_evidence[{index}] artifact measurement window mismatch") + + +def validate_release_bundle_json_artifact_payload(bundle_path: Path, path: Path, source_revision: str, gate: str, field: str, + run_id: str, window_id: str, + artifact_kind: str | None = None, + evidence: dict[str, object] | None = None) -> None: + payload = read_json(path) + prefix = f"{gate}.{field}" + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(payload.get(marker) is not True, f"{prefix} JSON artifact is {marker}") + require(payload.get("evidence_type") == "measured", f"{prefix} JSON artifact must be measured") + require(payload.get("source_revision") == source_revision, f"{prefix} JSON artifact source revision mismatch") + require(payload.get("run_id") == run_id, f"{prefix} JSON artifact run_id mismatch") + require(payload.get("measurement_window_id") == window_id, f"{prefix} JSON artifact measurement window mismatch") + require(payload.get("gate") == gate, f"{prefix} JSON artifact gate mismatch") + require(payload.get("field") == field, f"{prefix} JSON artifact field mismatch") + if artifact_kind is not None: + require(payload.get("artifact_kind") == artifact_kind, f"{prefix} JSON artifact kind mismatch") + if field == "profile_evidence": + validate_profile_json_artifact_payload(bundle_path, path, payload, gate, field, artifact_kind) + mrf_artifact_kind = SCANNER_HEAL_RELEASE_MRF_ARTIFACT_KINDS.get((gate, field)) + if mrf_artifact_kind is not None: + require(payload.get("artifact_kind") == mrf_artifact_kind, + f"{prefix} JSON artifact kind must be {mrf_artifact_kind}") + mirror_fields = release_bundle_json_artifact_mirrored_fields(gate, field) + for mirror_field in mirror_fields: + require(mirror_field in payload, f"{prefix} JSON artifact missing {mirror_field}") + if not mirror_fields: + return + validate_release_bundle_domain_evidence(gate, field, payload) + if gate == "G14": + validate_g14_json_artifact_payload(bundle_path, payload, evidence, gate, field, source_revision, window_id) + if gate in ("G03", "G09", "R-L"): + versions = payload.get("versions") + require(isinstance(versions, list) and + len(set(versions)) >= 2 and + all(isinstance(version, str) and re.fullmatch(r"[0-9a-f]{40}", version) is not None + for version in versions), + f"{prefix} JSON artifact requires mixed-version evidence") + require(source_revision in versions, f"{prefix} JSON artifact versions omit tested source revision") + expected_role = SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[(gate, field)] + require(payload.get("mixed_version_role") == expected_role, + f"{prefix} JSON artifact mixed-version role must be {expected_role}") + if gate in ("G04", "G07", "R-E", "R-L"): + crash_points = payload.get("crash_points") + require(isinstance(crash_points, list) and crash_points, + f"{prefix} JSON artifact requires crash-boundary evidence") + if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS: + evidence_integer(payload.get("replayed_records"), f"{prefix} JSON artifact replayed_records", 1, 2**63 - 1) + require(payload.get("responsibility_anchor_retained") is True, + f"{prefix} JSON artifact requires retained MRF responsibility anchors") + require(payload.get("successor_snapshot_published") is True, + f"{prefix} JSON artifact requires successor snapshot publication evidence") + if gate == "P4" and field == "mrf_cleanup_gc_soak_evidence": + release_bundle_exact_strings( + payload.get("cleanup_gc_cases"), + SCANNER_HEAL_RELEASE_MRF_CLEANUP_GC_SOAK_CASES, + f"{prefix} JSON artifact cleanup_gc_cases", + ) + require(payload.get("verified_idle_gc_observed") is True, + f"{prefix} JSON artifact requires verified idle GC evidence") + require(payload.get("pending_responsibilities_after_gc") == 0, + f"{prefix} JSON artifact requires zero pending responsibilities after GC") + require(payload.get("stale_journals_after_gc") == 0, + f"{prefix} JSON artifact requires zero stale journals after GC") + if gate == "G07": + case_field = { + "mrf_responsibility_oracle": "mrf_responsibility_cases", + "commit_boundary_crash_matrix": "commit_crash_cases", + }[field] + cases = evidence_string_list(payload.get(case_field), f"{prefix} JSON artifact {case_field}") + missing_cases = sorted(set(SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES[field]) - set(cases)) + require(not missing_cases, f"{prefix} JSON artifact missing cases: {', '.join(missing_cases)}") + if gate == "G08": + case_field = { + "mrf_capacity_evidence": "capacity_cases", + "disk_full_matrix": "disk_full_cases", + "replica_loss_matrix": "replica_loss_cases", + }[field] + cases = evidence_string_list(payload.get(case_field), f"{prefix} JSON artifact {case_field}") + missing_cases = sorted(set(SCANNER_HEAL_RELEASE_G08_REQUIRED_CASES[field]) - set(cases)) + require(not missing_cases, f"{prefix} JSON artifact missing cases: {', '.join(missing_cases)}") + + +def release_bundle_bool_true(value: object, name: str) -> None: + require(value is True, f"{name} must be true") + + +def release_bundle_exact_strings(value: object, expected: tuple[str, ...], name: str) -> None: + require(isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value), + f"{name} must list measured cases") + observed = set(value) + require(len(observed) == len(value), f"{name} has duplicate cases") + missing = sorted(set(expected) - observed) + require(not missing, f"{name} missing cases: {', '.join(missing)}") + unknown = sorted(observed - set(expected)) + require(not unknown, f"{name} has unknown cases: {', '.join(unknown)}") + + +def release_bundle_number(value: object, name: str, minimum: int | float = 0) -> float: + return number(value, name, minimum) + + +def validate_release_bundle_domain_evidence(gate: str, field: str, evidence: dict[str, object]) -> None: + if gate == "G01": + if field == "root_authority_evidence": + release_bundle_exact_strings( + evidence.get("root_authority_cases"), + SCANNER_HEAL_RELEASE_G01_ROOT_AUTHORITY_CASES, + f"{gate}.{field}.root_authority_cases", + ) + release_bundle_bool_true(evidence.get("root_cas_observed"), f"{gate}.{field}.root_cas_observed") + release_bundle_bool_true(evidence.get("root_readback_observed"), f"{gate}.{field}.root_readback_observed") + release_bundle_bool_true(evidence.get("incomplete_root_rejected"), + f"{gate}.{field}.incomplete_root_rejected") + release_bundle_bool_true(evidence.get("stale_root_rejected"), f"{gate}.{field}.stale_root_rejected") + if field == "quota_authority_evidence": + release_bundle_exact_strings( + evidence.get("quota_authority_cases"), + SCANNER_HEAL_RELEASE_G01_QUOTA_AUTHORITY_CASES, + f"{gate}.{field}.quota_authority_cases", + ) + release_bundle_bool_true(evidence.get("quota_floor_readback_observed"), + f"{gate}.{field}.quota_floor_readback_observed") + release_bundle_bool_true(evidence.get("over_limit_put_rejected"), + f"{gate}.{field}.over_limit_put_rejected") + release_bundle_bool_true(evidence.get("rejected_object_invisible"), + f"{gate}.{field}.rejected_object_invisible") + release_bundle_bool_true(evidence.get("quota_fails_closed_without_authority"), + f"{gate}.{field}.quota_fails_closed_without_authority") + + if gate == "G02": + if field == "bounded_checkpoint_oracle": + release_bundle_bool_true(evidence.get("checkpoint_progress_bounded"), + f"{gate}.{field}.checkpoint_progress_bounded") + budget = evidence_integer(evidence.get("raw_entry_budget"), f"{gate}.{field}.raw_entry_budget", 1, 4096) + max_raw = evidence_integer(evidence.get("max_raw_entries_per_round"), + f"{gate}.{field}.max_raw_entries_per_round", 1, 4096) + max_objects = evidence_integer(evidence.get("max_objects_processed_per_round"), + f"{gate}.{field}.max_objects_processed_per_round", 1, 4096) + require(max_raw <= budget, f"{gate}.{field} raw entries exceed fixed budget") + require(max_objects <= budget, f"{gate}.{field} processed objects exceed fixed budget") + release_bundle_bool_true(evidence.get("bounded_work_quantum_observed"), + f"{gate}.{field}.bounded_work_quantum_observed") + release_bundle_bool_true(evidence.get("durable_checkpoint_committed"), + f"{gate}.{field}.durable_checkpoint_committed") + release_bundle_bool_true(evidence.get("no_unbounded_tail"), f"{gate}.{field}.no_unbounded_tail") + if field == "independent_version_inventory": + release_bundle_bool_true(evidence.get("independent_version_inventory_observed"), + f"{gate}.{field}.independent_version_inventory_observed") + objects_expected = evidence_integer(evidence.get("objects_expected"), + f"{gate}.{field}.objects_expected", 1, 2**63 - 1) + objects_retained = evidence_integer(evidence.get("objects_retained"), + f"{gate}.{field}.objects_retained", 1, 2**63 - 1) + versions_retained = evidence_integer(evidence.get("versions_retained"), + f"{gate}.{field}.versions_retained", 1, 2**63 - 1) + bytes_retained = evidence_integer(evidence.get("bytes_retained"), + f"{gate}.{field}.bytes_retained", 1, 2**63 - 1) + require(objects_retained == objects_expected, + f"{gate}.{field} retained object inventory must match expected objects") + require(versions_retained == objects_expected, + f"{gate}.{field} retained version inventory must match expected objects") + require(bytes_retained == objects_expected, + f"{gate}.{field} retained byte inventory must match expected objects") + + if gate == "G03": + release_bundle_exact_strings( + evidence.get("scoped_ack_cases"), + SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES[field], + f"{gate}.{field}.scoped_ack_cases", + ) + for required in SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS[field]: + release_bundle_bool_true(evidence.get(required), f"{gate}.{field}.{required}") + + if gate == "G04" and field == "root_floor_intent_crash_evidence": + release_bundle_exact_strings( + evidence.get("durable_intent_cases"), + SCANNER_HEAL_RELEASE_CRASH_BOUNDARY_FIELDS[(gate, field)], + f"{gate}.{field}.durable_intent_cases", + ) + release_bundle_bool_true(evidence.get("persist_failure_blocks_acceptance"), + f"{gate}.{field}.persist_failure_blocks_acceptance") + + if gate == "G12": + release_bundle_exact_strings( + evidence.get("quota_path_cases"), + SCANNER_HEAL_RELEASE_G12_QUOTA_PATH_CASES[field], + f"{gate}.{field}.quota_path_cases", + ) + + if gate == "G05": + if field == "per_object_outcome_oracle": + release_bundle_exact_strings( + evidence.get("per_object_outcome_cases"), + SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES, + f"{gate}.{field}.per_object_outcome_cases", + ) + outcomes = evidence.get("outcome_counts") + require(isinstance(outcomes, dict), f"{gate}.{field} missing outcome counts") + for outcome in ("repaired", "healthy", "skipped", "failed"): + evidence_integer(outcomes.get(outcome), f"{gate}.{field}.outcome_counts.{outcome}", 1, 2**63 - 1) + release_bundle_bool_true(evidence.get("status_matches_object_oracle"), + f"{gate}.{field}.status_matches_object_oracle") + if field == "terminal_retention_bounds": + release_bundle_exact_strings( + evidence.get("terminal_retention_cases"), + SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES, + f"{gate}.{field}.terminal_retention_cases", + ) + retention_window = evidence_integer( + evidence.get("terminal_retention_window_seconds"), + f"{gate}.{field}.terminal_retention_window_seconds", + 1, + 86400, + ) + max_age = evidence_integer( + evidence.get("max_terminal_record_age_seconds"), + f"{gate}.{field}.max_terminal_record_age_seconds", + 0, + 86400, + ) + require(max_age <= retention_window, f"{gate}.{field}.max_terminal_record_age_seconds exceeds retention window") + evidence_integer(evidence.get("terminal_records_pruned_after_window"), + f"{gate}.{field}.terminal_records_pruned_after_window", 1, 2**63 - 1) + + if gate == "G06": + if field == "concurrent_status_evidence": + release_bundle_exact_strings( + evidence.get("concurrent_status_cases"), + SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES, + f"{gate}.{field}.concurrent_status_cases", + ) + evidence_integer(evidence.get("status_samples"), f"{gate}.{field}.status_samples", 2, 2**63 - 1) + release_bundle_bool_true(evidence.get("all_status_responses_http_success"), + f"{gate}.{field}.all_status_responses_http_success") + release_bundle_bool_true(evidence.get("partial_status_reports_degraded"), + f"{gate}.{field}.partial_status_reports_degraded") + if field == "legacy_client_compatibility": + release_bundle_exact_strings( + evidence.get("legacy_client_cases"), + SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES, + f"{gate}.{field}.legacy_client_cases", + ) + release_bundle_bool_true(evidence.get("rustfs_and_minio_paths_compatible"), + f"{gate}.{field}.rustfs_and_minio_paths_compatible") + release_bundle_bool_true(evidence.get("empty_body_status_requests_accepted"), + f"{gate}.{field}.empty_body_status_requests_accepted") + if field == "truncation_behavior": + release_bundle_exact_strings( + evidence.get("truncation_cases"), + SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES, + f"{gate}.{field}.truncation_cases", + ) + release_bundle_bool_true(evidence.get("truncated_payloads_rejected"), + f"{gate}.{field}.truncated_payloads_rejected") + evidence_integer(evidence.get("max_status_payload_bytes"), f"{gate}.{field}.max_status_payload_bytes", 1, 2**20) + + if gate == "R-E": + if field == "fixed_budget_restart_evidence": + release_bundle_bool_true(evidence.get("fixed_budget_restart_converged"), + f"{gate}.{field}.fixed_budget_restart_converged") + evidence_integer(evidence.get("restart_rounds"), f"{gate}.{field}.restart_rounds", 2, 64) + evidence_integer(evidence.get("raw_entry_budget"), f"{gate}.{field}.raw_entry_budget", 1, 4096) + release_bundle_bool_true(evidence.get("no_unbudgeted_final_sweep"), + f"{gate}.{field}.no_unbudgeted_final_sweep") + if field == "enumeration_evidence": + release_bundle_bool_true(evidence.get("raw_enumeration_observed"), + f"{gate}.{field}.raw_enumeration_observed") + release_bundle_bool_true(evidence.get("durable_raw_page_commit_observed"), + f"{gate}.{field}.durable_raw_page_commit_observed") + release_bundle_bool_true(evidence.get("raw_page_index_complete"), + f"{gate}.{field}.raw_page_index_complete") + release_bundle_bool_true(evidence.get("enumeration_frontier_retained"), + f"{gate}.{field}.enumeration_frontier_retained") + if field == "classification_evidence": + release_bundle_bool_true(evidence.get("classification_observed"), + f"{gate}.{field}.classification_observed") + objects_processed = evidence_integer(evidence.get("objects_processed"), + f"{gate}.{field}.objects_processed", 1, 2**63 - 1) + objects_retained = evidence_integer(evidence.get("objects_retained"), + f"{gate}.{field}.objects_retained", 1, 2**63 - 1) + versions_retained = evidence_integer(evidence.get("versions_retained"), + f"{gate}.{field}.versions_retained", 1, 2**63 - 1) + bytes_retained = evidence_integer(evidence.get("bytes_retained"), + f"{gate}.{field}.bytes_retained", 1, 2**63 - 1) + require(objects_processed <= objects_retained, + f"{gate}.{field} processed objects exceed retained objects") + require(versions_retained == objects_retained, + f"{gate}.{field} version inventory must match retained objects") + require(bytes_retained == objects_retained, + f"{gate}.{field} byte inventory must match retained objects") + + if gate == "G09": + release_bundle_exact_strings( + evidence.get("mixed_version_cases"), + SCANNER_HEAL_RELEASE_MIXED_VERSION_CASES[field], + f"{gate}.{field}.mixed_version_cases", + ) + if field == "rollback_payload_evidence": + release_bundle_bool_true(evidence.get("rollback_payload_replayed"), + f"{gate}.{field}.rollback_payload_replayed") + + if gate == "G10": + if field == "scheduler_bound_evidence": + release_bundle_exact_strings( + evidence.get("scheduler_bounds"), + SCANNER_HEAL_RELEASE_SCHEDULER_BOUNDS, + f"{gate}.{field}.scheduler_bounds", + ) + release_bundle_bool_true(evidence.get("duplicate_task_bound_observed"), + f"{gate}.{field}.duplicate_task_bound_observed") + if field == "pressure_recovery_evidence": + metrics = evidence.get("pressure_metrics") + require(isinstance(metrics, dict), f"{gate}.{field} missing pressure metrics") + for metric in SCANNER_HEAL_RELEASE_PRESSURE_METRICS: + release_bundle_number(metrics.get(metric), f"{gate}.{field}.{metric}") + evidence_integer(metrics.get("foreground_pressure_samples"), + f"{gate}.{field}.foreground_pressure_samples", 1, 2**63 - 1) + evidence_integer(metrics.get("foreground_pressure_high_samples"), + f"{gate}.{field}.foreground_pressure_high_samples", 1, 2**63 - 1) + + if gate == "G11": + if field == "maintenance_producer_matrix": + release_bundle_exact_strings( + evidence.get("producer_identities"), + SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES, + f"{gate}.{field}.producer_identities", + ) + release_bundle_exact_strings( + evidence.get("producer_families"), + SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES, + f"{gate}.{field}.producer_families", + ) + release_bundle_exact_strings( + evidence.get("matrix_cases"), + SCANNER_HEAL_RELEASE_G11_REQUIRED_CASES[field], + f"{gate}.{field}.matrix_cases", + ) + release_bundle_bool_true(evidence.get("durable_identity_observed"), + f"{gate}.{field}.durable_identity_observed") + release_bundle_bool_true(evidence.get("durable_journal_replay_observed"), + f"{gate}.{field}.durable_journal_replay_observed") + release_bundle_bool_true(evidence.get("durable_journal_hydration_bound"), + f"{gate}.{field}.durable_journal_hydration_bound") + release_bundle_bool_true(evidence.get("generation_window_bound"), + f"{gate}.{field}.generation_window_bound") + release_bundle_bool_true(evidence.get("restart_gap_absent"), f"{gate}.{field}.restart_gap_absent") + release_bundle_bool_true(evidence.get("overflow_absent"), f"{gate}.{field}.overflow_absent") + release_bundle_bool_true(evidence.get("ack_clear_generation_observed"), + f"{gate}.{field}.ack_clear_generation_observed") + release_bundle_exact_strings( + evidence.get("invalid_journal_inputs_rejected"), + SCANNER_HEAL_DURABLE_JOURNAL_FAIL_CLOSED_CASES, + f"{gate}.{field}.invalid_journal_inputs_rejected", + ) + if field == "complete_producer_inventory": + release_bundle_exact_strings( + evidence.get("required_producer_identities"), + SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES, + f"{gate}.{field}.required_producer_identities", + ) + release_bundle_exact_strings( + evidence.get("observed_producer_identities"), + SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES, + f"{gate}.{field}.observed_producer_identities", + ) + release_bundle_exact_strings( + evidence.get("required_producer_families"), + SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES, + f"{gate}.{field}.required_producer_families", + ) + release_bundle_exact_strings( + evidence.get("observed_producer_families"), + SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES, + f"{gate}.{field}.observed_producer_families", + ) + require(evidence.get("missing_producer_identities") == [], + f"{gate}.{field} requires zero missing producer identities") + release_bundle_bool_true(evidence.get("unknown_producer_excluded"), + f"{gate}.{field}.unknown_producer_excluded") + + if gate == "G13": + if field == "quorum_minus_one_matrix": + release_bundle_exact_strings( + evidence.get("quorum_cases"), + SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES[field], + f"{gate}.{field}.quorum_cases", + ) + release_bundle_bool_true(evidence.get("no_success_at_quorum_minus_one"), + f"{gate}.{field}.no_success_at_quorum_minus_one") + release_bundle_bool_true(evidence.get("exact_quorum_restored"), f"{gate}.{field}.exact_quorum_restored") + if field == "unknown_disk_remount_matrix": + release_bundle_exact_strings( + evidence.get("remount_cases"), + SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES[field], + f"{gate}.{field}.remount_cases", + ) + release_bundle_bool_true(evidence.get("unknown_disks_excluded"), + f"{gate}.{field}.unknown_disks_excluded") + release_bundle_bool_true(evidence.get("remounted_disks_revalidated"), + f"{gate}.{field}.remounted_disks_revalidated") + release_bundle_bool_true(evidence.get("stale_incarnation_rejected"), + f"{gate}.{field}.stale_incarnation_rejected") + if field == "object_lock_dry_run_grace_evidence": + release_bundle_exact_strings( + evidence.get("grace_cases"), + SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES[field], + f"{gate}.{field}.grace_cases", + ) + release_bundle_bool_true(evidence.get("object_lock_denials_preserved"), + f"{gate}.{field}.object_lock_denials_preserved") + evidence_integer(evidence.get("dry_run_mutation_count"), f"{gate}.{field}.dry_run_mutation_count", 0, 0) + release_bundle_bool_true(evidence.get("grace_outcomes_retained"), + f"{gate}.{field}.grace_outcomes_retained") + + if gate == "P1": + if field == "cold_walk_share_measurement": + share = release_bundle_number(evidence.get("cold_walk_share"), f"{gate}.{field}.cold_walk_share") + require(share <= 1, f"{gate}.{field}.cold_walk_share exceeds one") + evidence_integer(evidence.get("walk_objects"), f"{gate}.{field}.walk_objects", 1, 2**63 - 1) + evidence_integer(evidence.get("cold_walk_objects"), f"{gate}.{field}.cold_walk_objects", 0, 2**63 - 1) + if field == "foreground_latency_throughput_measurement": + for metric in ("foreground_p95_ms", "foreground_p99_ms", "throughput_ops", "error_rate"): + release_bundle_number(evidence.get(metric), f"{gate}.{field}.{metric}") + if field == "profile_evidence": + for metric in ("allocation_bytes", "rss_peak_bytes", "save_operations", "saved_bytes"): + evidence_integer(evidence.get(metric), f"{gate}.{field}.{metric}", 1, 2**63 - 1) + + if gate == "P3": + if field == "two_hour_pressure_measurement": + release_bundle_exact_strings(evidence.get("abba_legs"), ("A1", "B1", "B2", "A2"), + f"{gate}.{field}.abba_legs") + release_bundle_exact_strings(evidence.get("scenarios"), SCENARIOS, f"{gate}.{field}.scenarios") + for metric in ("foreground_p95_ms", "foreground_p99_ms", "throughput_ops"): + release_bundle_number(evidence.get(metric), f"{gate}.{field}.{metric}", 1) + if field == "heal_capacity_measurement": + capacity = evidence.get("heal_capacity") + require(isinstance(capacity, dict), f"{gate}.{field} missing heal capacity") + for metric in ("objects", "versions", "bytes", "completed_objects"): + evidence_integer(capacity.get(metric), f"{gate}.{field}.{metric}", 1, 2**63 - 1) + if field == "recovery_window_measurement": + release_bundle_exact_strings( + evidence.get("fault_modes"), + ("process-restart", "process-crash-restart"), + f"{gate}.{field}.fault_modes", + ) + for metric in ("recovery_p95_ms", "recovery_p99_ms"): + release_bundle_number(evidence.get(metric), f"{gate}.{field}.{metric}", 1) + + if gate == "P4" and field == "retained_responsibility_evidence": + release_bundle_exact_strings( + evidence.get("retained_responsibility_cases"), + SCANNER_HEAL_RELEASE_P4_RETAINED_RESPONSIBILITY_CASES, + f"{gate}.{field}.retained_responsibility_cases", + ) + retention_window = evidence_integer( + evidence.get("retention_window_seconds"), + f"{gate}.{field}.retention_window_seconds", + 7200, + 86400, + ) + duration = evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400) + require(duration >= retention_window, f"{gate}.{field} duration must cover retention window") + release_bundle_bool_true(evidence.get("idle_cleanup_observed"), f"{gate}.{field}.idle_cleanup_observed") + release_bundle_bool_true( + evidence.get("verified_proof_discharge_observed"), + f"{gate}.{field}.verified_proof_discharge_observed", + ) + + if gate == "R-D": + if field == "manager_disposition_evidence": + release_bundle_exact_strings( + evidence.get("manager_disposition_cases"), + SCANNER_HEAL_RELEASE_RD_MANAGER_CASES, + f"{gate}.{field}.manager_disposition_cases", + ) + release_bundle_bool_true(evidence.get("manager_dispositions_are_terminal"), + f"{gate}.{field}.manager_dispositions_are_terminal") + if field == "event_disposition_evidence": + release_bundle_exact_strings( + evidence.get("event_disposition_cases"), + SCANNER_HEAL_RELEASE_RD_EVENT_CASES, + f"{gate}.{field}.event_disposition_cases", + ) + release_bundle_bool_true(evidence.get("events_correlate_to_manager_dispositions"), + f"{gate}.{field}.events_correlate_to_manager_dispositions") + if field == "ledger_disposition_evidence": + release_bundle_exact_strings( + evidence.get("ledger_disposition_cases"), + SCANNER_HEAL_RELEASE_RD_LEDGER_CASES, + f"{gate}.{field}.ledger_disposition_cases", + ) + release_bundle_bool_true(evidence.get("ledger_correlates_to_events"), + f"{gate}.{field}.ledger_correlates_to_events") + release_bundle_bool_true(evidence.get("ledger_replay_preserves_terminal_disposition"), + f"{gate}.{field}.ledger_replay_preserves_terminal_disposition") + if field == "grace_handling": + release_bundle_exact_strings( + evidence.get("grace_cases"), + SCANNER_HEAL_RELEASE_RD_GRACE_CASES, + f"{gate}.{field}.grace_cases", + ) + evidence_integer(evidence.get("grace_window_seconds"), f"{gate}.{field}.grace_window_seconds", 1, 86400) + release_bundle_bool_true(evidence.get("grace_retention_observed"), + f"{gate}.{field}.grace_retention_observed") + release_bundle_bool_true(evidence.get("grace_expiry_pruned_terminal_records"), + f"{gate}.{field}.grace_expiry_pruned_terminal_records") + + if gate == "R-L": + if field == "legacy_source_conflict_evidence": + release_bundle_exact_strings( + evidence.get("legacy_source_conflict_cases"), + SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES[field], + f"{gate}.{field}.legacy_source_conflict_cases", + ) + release_bundle_bool_true(evidence.get("source_conflicts_rejected"), + f"{gate}.{field}.source_conflicts_rejected") + release_bundle_bool_true(evidence.get("takeover_identity_bound"), + f"{gate}.{field}.takeover_identity_bound") + release_bundle_bool_true(evidence.get("legacy_checksum_gap_rejected"), + f"{gate}.{field}.legacy_checksum_gap_rejected") + if field == "migration_gap_evidence": + release_bundle_exact_strings( + evidence.get("migration_gap_cases"), + SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES[field], + f"{gate}.{field}.migration_gap_cases", + ) + release_bundle_bool_true(evidence.get("migration_gap_closed"), + f"{gate}.{field}.migration_gap_closed") + release_bundle_bool_true(evidence.get("legacy_sources_fail_closed"), + f"{gate}.{field}.legacy_sources_fail_closed") + release_bundle_bool_true(evidence.get("prior_responsibilities_inherited"), + f"{gate}.{field}.prior_responsibilities_inherited") + if field == "crash_safe_source_retirement_evidence": + release_bundle_exact_strings( + evidence.get("source_retirement_cases"), + SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES[field], + f"{gate}.{field}.source_retirement_cases", + ) + release_bundle_bool_true(evidence.get("source_retirement_is_crash_safe"), + f"{gate}.{field}.source_retirement_is_crash_safe") + release_bundle_bool_true(evidence.get("old_source_retained_until_successor"), + f"{gate}.{field}.old_source_retained_until_successor") + release_bundle_bool_true(evidence.get("recovered_pending_migration"), + f"{gate}.{field}.recovered_pending_migration") + + +def validate_release_bundle_artifact(bundle_path: Path, source_revision: str, gate: str, field: str, + evidence: dict[str, object]) -> str: + require(evidence.get("evidence_type") == "measured", f"{gate}.{field} must be measured evidence") + require(evidence.get("source_revision") == source_revision, f"{gate}.{field} source revision mismatch") + run_id = evidence_string(evidence.get("run_id"), f"{gate}.{field}.run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + window_id = evidence_string(evidence.get("measurement_window_id"), f"{gate}.{field}.measurement_window_id", + r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + require(window_id != run_id, f"{gate}.{field} must separate run and measurement-window identities") + started = evidence_timestamp(evidence.get("started_at"), f"{gate}.{field}.started_at") + finished = evidence_timestamp(evidence.get("finished_at"), f"{gate}.{field}.finished_at") + require(started <= finished, f"{gate}.{field} evidence timestamps are inverted") + command = evidence.get("command") + require(isinstance(command, list) and command and + all(isinstance(part, str) and part.strip() for part in command), + f"{gate}.{field} missing command provenance") + artifact_format = evidence_string(evidence.get("artifact_format"), f"{gate}.{field}.artifact_format", + r"[A-Za-z0-9][A-Za-z0-9._+:-]{1,63}") + artifact = release_bundle_artifact_path(bundle_path, evidence.get("artifact"), gate, field) + require(sha(evidence.get("sha256")) and digest(artifact) == evidence["sha256"], f"{gate}.{field} artifact hash mismatch") + if is_json_artifact_format(artifact_format): + validate_release_bundle_json_artifact_payload( + bundle_path, + artifact, + source_revision, + gate, + field, + run_id, + window_id, + evidence=evidence, + ) + summary = evidence.get("summary") + require(isinstance(summary, str) and summary.strip(), f"{gate}.{field} missing human summary") + if gate.startswith("P"): + duration = evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400) + require((finished - started).total_seconds() + 1 >= duration, f"{gate}.{field} duration exceeds run window") + require(duration >= 900, f"{gate}.{field} requires at least 900 seconds") + if gate == "P3" and field == "two_hour_pressure_measurement": + require(duration >= 7200, f"{gate}.{field} requires at least two hours") + if gate == "P4" and field == "mrf_cleanup_gc_soak_evidence": + require(duration >= 7200, f"{gate}.{field} requires at least two hours") + elif "duration_seconds" in evidence: + evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400) + if gate == "G10": + if field == "scheduler_bound_evidence": + evidence_integer(evidence.get("max_deferred_items"), f"{gate}.{field}.max_deferred_items", 1, 2**31 - 1) + evidence_integer(evidence.get("max_deferred_bytes"), f"{gate}.{field}.max_deferred_bytes", 1, 2**63 - 1) + evidence_integer(evidence.get("max_retry_age_seconds"), f"{gate}.{field}.max_retry_age_seconds", 1, 86400) + evidence_integer(evidence.get("duplicate_task_count"), f"{gate}.{field}.duplicate_task_count", 0, 0) + if field == "pressure_recovery_evidence": + require(evidence.get("pressure_pacing_engaged") is True, + f"{gate}.{field} requires pressure pacing engagement") + evidence_integer(evidence.get("recovery_window_seconds"), f"{gate}.{field}.recovery_window_seconds", 1, 86400) + evidence_integer(evidence.get("lock_hold_p95_ms"), f"{gate}.{field}.lock_hold_p95_ms", 0, 2**31 - 1) + evidence_integer(evidence.get("foreground_latency_p95_ms"), + f"{gate}.{field}.foreground_latency_p95_ms", 1, 2**31 - 1) + if gate in ("G01", "G11", "G12", "G13", "R-L"): + validate_release_bundle_domain_evidence(gate, field, evidence) + if gate == "P1" and field == "foreground_latency_throughput_measurement": + evidence_integer(evidence.get("foreground_latency_p95_ms"), + f"{gate}.{field}.foreground_latency_p95_ms", 1, 2**31 - 1) + evidence_integer(evidence.get("foreground_latency_p99_ms"), + f"{gate}.{field}.foreground_latency_p99_ms", 1, 2**31 - 1) + evidence_integer(evidence.get("throughput_ops_per_second"), + f"{gate}.{field}.throughput_ops_per_second", 1, 2**31 - 1) + evidence_integer(evidence.get("error_count"), f"{gate}.{field}.error_count", 0, 0) + if gate == "P3": + if field == "two_hour_pressure_measurement": + require(evidence.get("fixed_offered_load") is True, f"{gate}.{field} requires fixed offered load") + evidence_integer(evidence.get("foreground_latency_p99_ms"), + f"{gate}.{field}.foreground_latency_p99_ms", 1, 2**31 - 1) + evidence_integer(evidence.get("attempt_cost_samples"), + f"{gate}.{field}.attempt_cost_samples", 1, 2**63 - 1) + if field == "heal_capacity_measurement": + evidence_integer(evidence.get("completed_heal_objects"), + f"{gate}.{field}.completed_heal_objects", 1, 2**63 - 1) + evidence_integer(evidence.get("duplicate_task_count"), f"{gate}.{field}.duplicate_task_count", 0, 0) + if field == "recovery_window_measurement": + evidence_integer(evidence.get("pressure_recovery_window_seconds"), + f"{gate}.{field}.pressure_recovery_window_seconds", 1, 86400) + evidence_integer(evidence.get("lock_hold_p95_ms"), f"{gate}.{field}.lock_hold_p95_ms", 0, 2**31 - 1) + if gate in ("G03", "G09", "R-L"): + versions = evidence.get("versions") + require(isinstance(versions, list) and + len(set(versions)) >= 2 and + all(isinstance(version, str) and re.fullmatch(r"[0-9a-f]{40}", version) is not None + for version in versions), + f"{gate}.{field} requires mixed-version evidence") + require(source_revision in versions, f"{gate}.{field} versions omit tested source revision") + expected_role = SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[(gate, field)] + require(evidence.get("mixed_version_role") == expected_role, + f"{gate}.{field} mixed-version role must be {expected_role}") + if gate in ("G04", "G07", "R-E", "R-L"): + crash_points = evidence.get("crash_points") + require(isinstance(crash_points, list) and crash_points, + f"{gate}.{field} requires crash-boundary evidence") + if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS: + evidence_integer(evidence.get("replayed_records"), f"{gate}.{field}.replayed_records", 1, 2**63 - 1) + require(evidence.get("responsibility_anchor_retained") is True, + f"{gate}.{field} requires retained MRF responsibility anchors") + require(evidence.get("successor_snapshot_published") is True, + f"{gate}.{field} requires successor snapshot publication evidence") + if gate == "P4" and field == "mrf_cleanup_gc_soak_evidence": + release_bundle_exact_strings( + evidence.get("cleanup_gc_cases"), + SCANNER_HEAL_RELEASE_MRF_CLEANUP_GC_SOAK_CASES, + f"{gate}.{field}.cleanup_gc_cases", + ) + require(evidence.get("verified_idle_gc_observed") is True, + f"{gate}.{field} requires verified idle GC evidence") + require(evidence.get("pending_responsibilities_after_gc") == 0, + f"{gate}.{field} requires zero pending responsibilities after GC") + require(evidence.get("stale_journals_after_gc") == 0, + f"{gate}.{field} requires zero stale journals after GC") + if field == "segment_activation_preflight": + require(evidence.get("production_activation") is True, + f"{gate}.{field} must prove production activation is enabled") + require(evidence.get("scanner_segment_reuse_activated") is True, + f"{gate}.{field} must prove the runtime activation gate is enabled") + evidence_exact_strings(evidence.get("proof_inputs"), + SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS, + f"{gate}.{field}.proof_inputs") + evidence_exact_strings(evidence.get("fail_closed_checks"), + SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS, + f"{gate}.{field}.fail_closed_checks") + if field == "cold_segment_reuse_measurement": + evidence_integer(evidence.get("hot_walked_segments"), f"{gate}.{field}.hot_walked_segments", 1, 2**63 - 1) + evidence_integer(evidence.get("cold_walked_segments"), f"{gate}.{field}.cold_walked_segments", 0, 0) + require(evidence.get("full_walk_oracle_equivalent") is True, + f"{gate}.{field} requires full-walk oracle equivalence") + require(evidence.get("published_root_equivalent") is True, + f"{gate}.{field} requires published-root equivalence") + if field == "post_stop_convergence_measurement": + require(evidence.get("writes_stopped") is True, + f"{gate}.{field} requires writes-stopped evidence") + require(evidence.get("last_mutation_observed") is True, + f"{gate}.{field} requires last-mutation observation") + require(evidence.get("first_complete_publication") is True, + f"{gate}.{field} requires first complete publication evidence") + samples = evidence_integer(evidence.get("post_stop_samples"), f"{gate}.{field}.post_stop_samples", 1, 2**31 - 1) + multiple = release_bundle_number(evidence.get("post_stop_work_multiple"), + f"{gate}.{field}.post_stop_work_multiple", 0) + limit = release_bundle_number(evidence.get("post_stop_work_multiple_limit"), + f"{gate}.{field}.post_stop_work_multiple_limit", 1) + require(multiple <= limit, f"{gate}.{field} exceeds post-stop work multiple limit") + raw_multiples = evidence.get("post_stop_work_multiples") + require(isinstance(raw_multiples, list) and len(raw_multiples) == samples, + f"{gate}.{field} requires measured post-stop work multiple samples") + observed = [release_bundle_number(value, f"{gate}.{field}.post_stop_work_multiples", 0) + for value in raw_multiples] + require(max(observed) == multiple, f"{gate}.{field} worst post-stop multiple mismatch") + if gate == "G07": + case_field = { + "mrf_responsibility_oracle": "mrf_responsibility_cases", + "commit_boundary_crash_matrix": "commit_crash_cases", + }[field] + cases = evidence_string_list(evidence.get(case_field), f"{gate}.{field}.{case_field}") + missing_cases = sorted(set(SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES[field]) - set(cases)) + require(not missing_cases, f"{gate}.{field} missing cases: {', '.join(missing_cases)}") + if gate == "G08": + case_field = { + "mrf_capacity_evidence": "capacity_cases", + "disk_full_matrix": "disk_full_cases", + "replica_loss_matrix": "replica_loss_cases", + }[field] + cases = evidence_string_list(evidence.get(case_field), f"{gate}.{field}.{case_field}") + missing_cases = sorted(set(SCANNER_HEAL_RELEASE_G08_REQUIRED_CASES[field]) - set(cases)) + require(not missing_cases, f"{gate}.{field} missing cases: {', '.join(missing_cases)}") + if gate == "G14": + if field == "same_window_field_evidence": + required_fields = set(SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS["G14"]) - {field} + same_window_fields = evidence.get("same_window_fields") + require(isinstance(same_window_fields, list) and + len(set(same_window_fields)) == len(same_window_fields) and + all(isinstance(item, str) and item in required_fields for item in same_window_fields), + "G14.same_window_field_evidence requires named G14 field coverage") + missing_same_window_fields = sorted(required_fields - set(same_window_fields)) + require(not missing_same_window_fields, + "G14.same_window_field_evidence missing fields: " + ", ".join(missing_same_window_fields)) + if field == "ec8_4_evidence": + topology = evidence.get("topology") + require(isinstance(topology, dict), "G14.ec8_4_evidence missing topology") + require(topology.get("erasure") == "EC8+4", "G14.ec8_4_evidence must record EC8+4") + require(evidence_integer(topology.get("nodes"), "G14 topology nodes", 3, 64) >= 3, + "G14.ec8_4_evidence requires at least three nodes") + require(evidence_integer(topology.get("drives_per_node"), "G14 topology drives", 4, 64) >= 4, + "G14.ec8_4_evidence requires at least four drives per node") + if field == "multi_set_evidence": + evidence_integer(evidence.get("sets"), "G14 multi_set_evidence.sets", 2, 1024) + if field == "multi_pool_evidence": + evidence_integer(evidence.get("pools"), "G14 multi_pool_evidence.pools", 2, 1024) + if field == "distributed_segment_invalidation_evidence": + require(evidence.get("invalidation_domain") == "distributed-ec", + "G14.distributed_segment_invalidation_evidence requires distributed EC invalidation") + require(evidence.get("distributed_ec_invalidation") is True, + "G14.distributed_segment_invalidation_evidence requires peer invalidation proof") + evidence_integer(evidence.get("peer_count"), "G14 distributed_segment_invalidation_evidence.peer_count", 3, 64) + require(evidence.get("same_window_remote_proof") is True, + "G14.distributed_segment_invalidation_evidence requires same-window remote proof") + require(evidence.get("all_peers_bound_to_generation_window") is True, + "G14.distributed_segment_invalidation_evidence requires peer generation-window binding") + if field == "profile_evidence": + evidence_integer(evidence.get("resolved_samples"), f"{gate}.{field}.resolved_samples", 1, 2**63 - 1) + profile_artifacts = evidence.get("profile_artifacts") + require(isinstance(profile_artifacts, dict), f"{gate}.{field} missing profile artifacts") + missing_artifacts = sorted(set(RELEASE_PROFILE_ARTIFACTS) - set(profile_artifacts)) + require(not missing_artifacts, + f"{gate}.{field} missing profile artifacts: {', '.join(missing_artifacts)}") + unknown_artifacts = sorted(set(profile_artifacts) - set(RELEASE_PROFILE_ARTIFACTS)) + require(not unknown_artifacts, + f"{gate}.{field} unknown profile artifacts: {', '.join(unknown_artifacts)}") + for artifact_kind in RELEASE_PROFILE_ARTIFACTS: + item = profile_artifacts[artifact_kind] + require(isinstance(item, dict), f"{gate}.{field}.{artifact_kind} must be an object") + artifact_field = f"{field}.{artifact_kind}" + artifact_path = release_bundle_artifact_path(bundle_path, item.get("artifact"), gate, artifact_field) + require(sha(item.get("sha256")) and digest(artifact_path) == item["sha256"], + f"{gate}.{artifact_field} artifact hash mismatch") + artifact_format = evidence_string(item.get("artifact_format"), f"{gate}.{artifact_field}.artifact_format", + r"[A-Za-z0-9][A-Za-z0-9._+:-]{1,63}") + require(item.get("source_revision") == source_revision, + f"{gate}.{artifact_field} source revision mismatch") + require(item.get("run_id") == run_id, f"{gate}.{artifact_field} run_id mismatch") + require(item.get("measurement_window_id") == window_id, + f"{gate}.{artifact_field} measurement window mismatch") + if is_json_artifact_format(artifact_format): + validate_release_bundle_json_artifact_payload( + bundle_path, + artifact_path, + source_revision, + gate, + field, + run_id, + window_id, + artifact_kind, + ) + if "resolved_samples" in item: + evidence_integer(item.get("resolved_samples"), f"{gate}.{artifact_field}.resolved_samples", + 0, 2**63 - 1) + validate_release_bundle_domain_evidence(gate, field, evidence) + return window_id + + +def validate_release_bundle_gate_windows(gate: str, field_windows: dict[str, str]) -> None: + if gate == "G14" or gate.startswith("P"): + windows = sorted(set(field_windows.values())) + require(len(windows) == 1, f"{gate} evidence fields must share one measurement window") + + +def validate_scanner_heal_release_bundle_gate(bundle_path: Path, source_revision: str, + requirements: dict[str, dict[str, object]], + raw_gates: dict[str, object], gate: str) -> list[str]: + gate_errors: list[str] = [] + requirement = requirements[gate] + gate_evidence = raw_gates.get(gate) + if not isinstance(gate_evidence, dict): + return ["missing gate evidence"] + if gate_evidence.get("status") != "pass": + gate_errors.append("gate status must be pass") + if gate_evidence.get("lane") != requirement["lane"]: + gate_errors.append("gate lane mismatch") + if gate_evidence.get("evidence_type") != "measured": + gate_errors.append("gate evidence type must be measured") + fields = gate_evidence.get("evidence_fields") + if not isinstance(fields, dict): + gate_errors.append("missing gate evidence fields") + fields = {} + required_fields = tuple(SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate]) + missing_fields = [field for field in required_fields if field not in fields] + if missing_fields: + gate_errors.append(f"missing required fields: {', '.join(missing_fields)}") + field_windows: dict[str, str] = {} + for field in required_fields: + if field not in fields: + continue + evidence = fields[field] + if not isinstance(evidence, dict): + gate_errors.append(f"{field} must be an object") + continue + for marker in ("fixture_only", "dry_run", "synthetic"): + if evidence.get(marker) is True: + gate_errors.append(f"{field} is {marker}") + continue + try: + field_windows[field] = validate_release_bundle_artifact(bundle_path, source_revision, gate, field, evidence) + except (OSError, KeyError, TypeError, ValueError) as error: + gate_errors.append(str(error)) + if not gate_errors: + try: + validate_release_bundle_gate_windows(gate, field_windows) + except ValueError as error: + gate_errors.append(str(error)) + return gate_errors + + +def scanner_heal_release_bundle_status(root: Path, bundle_path: Path) -> dict[str, object]: + """Validate a complete hard-gate evidence bundle without accepting synthetic claims.""" + registry = read_json(root / ".config/scanner-heal-required-tests.json") + requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry) + require(release_schema_capable, "scanner/heal release bundle requires schema 2 registry") + bundle_path = bundle_path.resolve() + bundle = read_json(bundle_path) + for marker in ("fixture_only", "dry_run", "synthetic"): + require(bundle.get(marker) is not True, f"scanner/heal release evidence bundle is {marker}") + require(bundle.get("schema") == 1, "unsupported scanner/heal release evidence bundle schema") + require(bundle.get("evidence") == "measured", "scanner/heal release evidence bundle must be measured") + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + source_revision = bundle.get("source_revision") + require(isinstance(source_revision, str) and re.fullmatch(r"[0-9a-f]{40}", source_revision) is not None and + source_revision == revision, + "scanner/heal release evidence source revision mismatch") + raw_gates = bundle.get("gates") + require(isinstance(raw_gates, dict), "scanner/heal release evidence bundle missing gates") + + verified: list[str] = [] + rejected: dict[str, list[str]] = {} + for gate in requirements: + gate_errors = validate_scanner_heal_release_bundle_gate(bundle_path, source_revision, requirements, raw_gates, gate) + if gate_errors: + rejected[gate] = gate_errors + else: + verified.append(gate) + + unknown = sorted(set(raw_gates) - set(requirements)) + if unknown: + rejected["unknown"] = [f"unknown gates: {', '.join(unknown)}"] + approved = not rejected and sorted(verified) == sorted(requirements) + return { + "schema": 1, + "decision": "approved" if approved else "blocked", + "release_approved": approved, + "release_schema_capable": release_schema_capable, + "verified_gates": sorted(verified), + "rejected_gates": rejected, + "pending_gates": [] if approved else sorted(gate for gate in requirements if gate not in verified), + "pending_lanes": [] if approved else pending_lanes, + } + + +def scanner_heal_release_bundle_gate_status(root: Path, bundle_path: Path, gate: str) -> dict[str, object]: + """Validate one measured release-bundle gate without approving the full release.""" + registry = read_json(root / ".config/scanner-heal-required-tests.json") + requirements, release_schema_capable, pending_lanes = scanner_heal_release_requirements(registry) + require(release_schema_capable, "scanner/heal release bundle gate requires schema 2 registry") + require(gate in requirements, f"unknown scanner/heal release gate: {gate}") + bundle_path = bundle_path.resolve() + bundle = read_json(bundle_path) + for marker in ("fixture_only", "dry_run", "synthetic"): + require(bundle.get(marker) is not True, f"scanner/heal release evidence bundle is {marker}") + require(bundle.get("schema") == 1, "unsupported scanner/heal release evidence bundle schema") + require(bundle.get("evidence") == "measured", "scanner/heal release evidence bundle must be measured") + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + source_revision = bundle.get("source_revision") + require(isinstance(source_revision, str) and re.fullmatch(r"[0-9a-f]{40}", source_revision) is not None and + source_revision == revision, + "scanner/heal release evidence source revision mismatch") + raw_gates = bundle.get("gates") + require(isinstance(raw_gates, dict), "scanner/heal release evidence bundle missing gates") + unknown = sorted(set(raw_gates) - set(requirements)) + require(not unknown, f"scanner/heal release evidence bundle has unknown gates: {', '.join(unknown)}") + + gate_errors = validate_scanner_heal_release_bundle_gate(bundle_path, source_revision, requirements, raw_gates, gate) + verified = not gate_errors + return { + "schema": 1, + "decision": "verified" if verified else "blocked", + "release_approved": False, + "release_schema_capable": release_schema_capable, + "verified_gate": gate if verified else None, + "rejected_gate": None if verified else gate, + "rejected_errors": gate_errors, + "pending_gates": sorted(item for item in requirements if item != gate), + "pending_lanes": pending_lanes, + } + + +def release_bundle_descriptor_path(descriptor_path: Path, raw_path: object, gate: str, field: str) -> Path: + require(isinstance(raw_path, str) and raw_path.strip(), f"{gate}.{field} missing source artifact") + path = Path(raw_path) + resolved = path if path.is_absolute() else descriptor_path.parent / path + resolved = resolved.resolve(strict=True) + require(resolved.is_file(), f"{gate}.{field} source artifact is missing") + require(resolved.stat().st_size > 0, f"{gate}.{field} source artifact is empty") + return resolved + + +def release_bundle_artifact_suffix(path: Path) -> str: + suffix = path.suffix + if re.fullmatch(r"\.[A-Za-z0-9._+-]{1,16}", suffix or "") is not None: + return suffix + return ".artifact" + + +def reject_release_bundle_markers(item: dict[str, object], label: str) -> None: + for marker in ("fixture_only", "dry_run", "synthetic"): + require(item.get(marker) is not True, f"{label} is {marker}") + + +def copy_release_bundle_artifact(descriptor_path: Path, bundle_dir: Path, evidence: dict[str, object], + gate: str, field: str, label: str) -> None: + source = release_bundle_descriptor_path(descriptor_path, evidence.get("artifact"), gate, field) + source_sha = digest(source) + if "sha256" in evidence: + require(sha(evidence.get("sha256")) and evidence["sha256"] == source_sha, + f"{gate}.{field} descriptor artifact hash mismatch") + artifact_dir = bundle_dir / "artifacts" + safe_label = re.sub(r"[^A-Za-z0-9._-]", "-", label) + target = artifact_dir / f"{safe_label}{release_bundle_artifact_suffix(source)}" + require(not target.exists(), f"{gate}.{field} duplicate assembled artifact path") + shutil.copyfile(source, target) + require(digest(target) == source_sha, f"{gate}.{field} assembled artifact hash mismatch") + evidence["artifact"] = target.relative_to(bundle_dir).as_posix() + evidence["sha256"] = source_sha + + +def copy_release_bundle_profile_raw_artifact(descriptor_path: Path, bundle_dir: Path, item: dict[str, object], + gate: str, field: str, artifact_kind: str) -> None: + wrapper = release_bundle_artifact_path(bundle_dir / "release-evidence.json", item.get("artifact"), gate, field) + payload = read_json(wrapper) + raw_source = release_bundle_descriptor_path( + descriptor_path, + payload.get("raw_profile_artifact"), + gate, + f"{field}.raw_profile", + ) + raw_sha = digest(raw_source) + require(sha(payload.get("raw_profile_sha256")) and payload["raw_profile_sha256"] == raw_sha, + f"{gate}.{field} descriptor raw profile hash mismatch") + raw_bytes = evidence_integer(payload.get("raw_profile_bytes"), f"{gate}.{field}.raw_profile_bytes", 1, 2**63 - 1) + require(raw_bytes == raw_source.stat().st_size, f"{gate}.{field} descriptor raw profile bytes mismatch") + artifact_dir = bundle_dir / "artifacts" + safe_label = re.sub(r"[^A-Za-z0-9._-]", "-", f"{gate}-{field}-{artifact_kind}-raw") + raw_target = artifact_dir / f"{safe_label}{release_bundle_artifact_suffix(raw_source)}" + require(not raw_target.exists(), f"{gate}.{field} duplicate assembled raw profile path") + shutil.copyfile(raw_source, raw_target) + require(digest(raw_target) == raw_sha, f"{gate}.{field} assembled raw profile hash mismatch") + payload["raw_profile_artifact"] = raw_target.relative_to(bundle_dir).as_posix() + payload["raw_profile_sha256"] = raw_sha + payload["raw_profile_bytes"] = raw_target.stat().st_size + write_json(wrapper, payload) + item["sha256"] = digest(wrapper) + + +def copy_release_bundle_g14_case_artifacts(descriptor_path: Path, bundle_dir: Path, evidence: dict[str, object], + gate: str, field: str) -> None: + wrapper = release_bundle_artifact_path(bundle_dir / "release-evidence.json", evidence.get("artifact"), gate, field) + payload = read_json(wrapper) + cases = payload.get("case_evidence") + require(isinstance(cases, list), f"{gate}.{field} descriptor case_evidence must be a list") + artifact_dir = bundle_dir / "artifacts" + for index, item in enumerate(cases): + require(isinstance(item, dict), f"{gate}.{field} descriptor case_evidence[{index}] must be an object") + case = evidence_string( + item.get("case"), + f"{gate}.{field}.case_evidence[{index}].case", + r"[A-Za-z0-9][A-Za-z0-9._:-]{1,127}", + ) + source = release_bundle_descriptor_path( + descriptor_path, + item.get("artifact"), + gate, + f"{field}.case_evidence.{case}", + ) + source_sha = digest(source) + require(sha(item.get("sha256")) and item["sha256"] == source_sha, + f"{gate}.{field} descriptor case_evidence[{index}] artifact hash mismatch") + safe_case = re.sub(r"[^A-Za-z0-9._-]", "-", case) + target = artifact_dir / f"{gate}-{field}-case-{safe_case}{release_bundle_artifact_suffix(source)}" + require(not target.exists(), f"{gate}.{field} duplicate assembled case evidence path") + shutil.copyfile(source, target) + require(digest(target) == source_sha, f"{gate}.{field} assembled case evidence hash mismatch") + item["artifact"] = target.relative_to(bundle_dir).as_posix() + item["sha256"] = source_sha + write_json(wrapper, payload) + evidence["case_evidence"] = payload["case_evidence"] + evidence["sha256"] = digest(wrapper) + + +def assemble_scanner_heal_release_bundle_descriptors( + root: Path, + descriptor_paths: list[Path], + directory: Path, +) -> tuple[Path, dict[str, object]]: + """Copy measured Scanner/Heal evidence descriptors into one validated release bundle.""" + require(not directory.exists(), "scanner/heal release bundle directory must be new") + require(descriptor_paths, "scanner/heal release bundle assembly requires at least one descriptor") + registry = read_json(root / ".config/scanner-heal-required-tests.json") + requirements, release_schema_capable, _ = scanner_heal_release_requirements(registry) + require(release_schema_capable, "scanner/heal release bundle assembly requires schema 2 registry") + + bundle_dir = directory.resolve() + artifact_dir = bundle_dir / "artifacts" + artifact_dir.mkdir(parents=True) + assembled_gates = {} + source_revision = None + for descriptor_index, raw_descriptor_path in enumerate(descriptor_paths): + descriptor_path = raw_descriptor_path.resolve(strict=True) + descriptor = read_json(descriptor_path) + label = f"scanner/heal release evidence descriptor {descriptor_index + 1}" + reject_release_bundle_markers(descriptor, label) + require(descriptor.get("schema") == 1, f"unsupported {label} schema") + require(descriptor.get("evidence") == "measured", f"{label} must be measured") + descriptor_revision = descriptor.get("source_revision") + require(isinstance(descriptor_revision, str) and re.fullmatch(r"[0-9a-f]{40}", descriptor_revision) is not None, + f"{label} source revision is invalid") + if source_revision is None: + source_revision = descriptor_revision + else: + require(descriptor_revision == source_revision, "scanner/heal release descriptor source revisions differ") + raw_gates = descriptor.get("gates") + require(isinstance(raw_gates, dict), f"{label} missing gates") + unknown_gates = sorted(set(raw_gates) - set(requirements)) + require(not unknown_gates, f"{label} has unknown gates: {', '.join(unknown_gates)}") + + for gate in sorted(raw_gates): + require(gate not in assembled_gates, f"scanner/heal release descriptor repeats gate: {gate}") + raw_gate = raw_gates[gate] + require(isinstance(raw_gate, dict), f"{gate} descriptor gate must be an object") + reject_release_bundle_markers(raw_gate, f"{gate} descriptor gate") + raw_fields = raw_gate.get("evidence_fields", raw_gate) + require(isinstance(raw_fields, dict), f"{gate} descriptor gate missing evidence fields") + required_fields = set(SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate]) + unknown_fields = sorted(set(raw_fields) - required_fields) + require(not unknown_fields, f"{gate} descriptor has unknown fields: {', '.join(unknown_fields)}") + fields = {} + for field in sorted(raw_fields): + raw_evidence = raw_fields[field] + require(isinstance(raw_evidence, dict), f"{gate}.{field} descriptor evidence must be an object") + evidence = json.loads(json.dumps(raw_evidence)) + reject_release_bundle_markers(evidence, f"{gate}.{field} descriptor evidence") + copy_release_bundle_artifact(descriptor_path, bundle_dir, evidence, gate, field, f"{gate}-{field}") + if gate == "G14": + copy_release_bundle_g14_case_artifacts(descriptor_path, bundle_dir, evidence, gate, field) + profile_artifacts = evidence.get("profile_artifacts") + if profile_artifacts is not None: + require(isinstance(profile_artifacts, dict), f"{gate}.{field} profile artifacts must be an object") + for artifact_kind, item in sorted(profile_artifacts.items()): + require(isinstance(item, dict), f"{gate}.{field}.{artifact_kind} descriptor profile artifact must be an object") + reject_release_bundle_markers(item, f"{gate}.{field}.{artifact_kind} descriptor profile artifact") + copy_release_bundle_artifact( + descriptor_path, + bundle_dir, + item, + gate, + f"{field}.{artifact_kind}", + f"{gate}-{field}-{artifact_kind}", + ) + if field == "profile_evidence": + copy_release_bundle_profile_raw_artifact( + descriptor_path, + bundle_dir, + item, + gate, + f"{field}.{artifact_kind}", + artifact_kind, + ) + fields[field] = evidence + assembled_gates[gate] = { + "status": "pass", + "lane": requirements[gate]["lane"], + "evidence_type": "measured", + "evidence_fields": fields, + } + + bundle = bundle_dir / "release-evidence.json" + write_json(bundle, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": assembled_gates, + }) + return bundle, scanner_heal_release_bundle_status(root, bundle) + + +def assemble_scanner_heal_release_bundle(root: Path, descriptor_path: Path, directory: Path) -> tuple[Path, dict[str, object]]: + """Copy one measured Scanner/Heal evidence descriptor into a validated release bundle.""" + return assemble_scanner_heal_release_bundle_descriptors(root, [descriptor_path], directory) + + +def write_scanner_heal_release_bundle_fixture(root: Path, directory: Path) -> Path: + """Write a non-approvable release bundle shape fixture for dry-run validation.""" + bundle_dir = directory.resolve() + bundle_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = bundle_dir / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + requirements, release_schema_capable, _ = scanner_heal_release_requirements(registry) + require(release_schema_capable, "scanner/heal release bundle fixture requires schema 2 registry") + source_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + baseline_revision = "0" * 39 + "1" + require(baseline_revision != source_revision, "scanner/heal release bundle fixture needs a distinct baseline") + started = datetime.now(timezone.utc).replace(microsecond=0) + gates = {} + for gate, requirement in requirements.items(): + fields = {} + for field in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate]: + artifact = artifact_dir / f"{gate}-{field}.json" + write_json(artifact, { + "schema": 1, + "fixture_only": True, + "gate": gate, + "field": field, + "source_revision": source_revision, + }) + duration = 60 + evidence = { + "fixture_only": True, + "artifact": artifact.relative_to(bundle_dir).as_posix(), + "sha256": digest(artifact), + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": f"{gate.lower()}-{field.replace('_', '-')}-fixture-run", + "measurement_window_id": f"{gate.lower()}-fixture-window", + "started_at": started.isoformat().replace("+00:00", "Z"), + "command": ["fixture-only", "scanner-heal-release-bundle", gate, field], + "artifact_format": "json", + "summary": f"fixture-only parser sample for {gate}.{field}; not release evidence", + } + if gate.startswith("P"): + duration = 900 + evidence["duration_seconds"] = duration + if gate == "P3" and field == "two_hour_pressure_measurement": + duration = 7200 + evidence["duration_seconds"] = duration + evidence["finished_at"] = (started + timedelta(seconds=duration)).isoformat().replace("+00:00", "Z") + if gate == "G10" and field == "scheduler_bound_evidence": + evidence.update({ + "max_deferred_items": 256, + "max_deferred_bytes": 262144, + "max_retry_age_seconds": 30, + "duplicate_task_count": 0, + }) + if gate == "G10" and field == "pressure_recovery_evidence": + evidence.update({ + "pressure_pacing_engaged": True, + "recovery_window_seconds": 5, + "lock_hold_p95_ms": 0, + "foreground_latency_p95_ms": 10, + }) + if gate == "P1" and field == "foreground_latency_throughput_measurement": + evidence.update({ + "foreground_latency_p95_ms": 10, + "foreground_latency_p99_ms": 20, + "throughput_ops_per_second": 100, + "error_count": 0, + }) + if gate == "P3" and field == "two_hour_pressure_measurement": + evidence.update({ + "fixed_offered_load": True, + "foreground_latency_p99_ms": 20, + "attempt_cost_samples": 1, + }) + if gate == "P3" and field == "heal_capacity_measurement": + evidence.update({"completed_heal_objects": 1, "duplicate_task_count": 0}) + if gate == "P3" and field == "recovery_window_measurement": + evidence.update({"pressure_recovery_window_seconds": 5, "lock_hold_p95_ms": 0}) + if gate == "G02" and field == "bounded_checkpoint_oracle": + evidence.update({ + "checkpoint_progress_bounded": True, + "raw_entry_budget": 8, + "max_raw_entries_per_round": 8, + "max_objects_processed_per_round": 8, + "bounded_work_quantum_observed": True, + "durable_checkpoint_committed": True, + "no_unbounded_tail": True, + }) + if gate == "G02" and field == "independent_version_inventory": + evidence.update({ + "independent_version_inventory_observed": True, + "objects_expected": 16, + "objects_retained": 16, + "versions_retained": 16, + "bytes_retained": 16, + }) + if gate == "R-E" and field == "fixed_budget_restart_evidence": + evidence.update({ + "fixed_budget_restart_converged": True, + "restart_rounds": 3, + "raw_entry_budget": 8, + "no_unbudgeted_final_sweep": True, + }) + if gate == "R-E" and field == "enumeration_evidence": + evidence.update({ + "raw_enumeration_observed": True, + "durable_raw_page_commit_observed": True, + "raw_page_index_complete": True, + "enumeration_frontier_retained": True, + }) + if gate == "R-E" and field == "classification_evidence": + evidence.update({ + "classification_observed": True, + "objects_processed": 16, + "objects_retained": 16, + "versions_retained": 16, + "bytes_retained": 16, + }) + if gate in ("G03", "G09", "R-L"): + evidence["versions"] = [baseline_revision, source_revision] + evidence["mixed_version_role"] = SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[(gate, field)] + if gate == "G03": + evidence["scoped_ack_cases"] = list(SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES[field]) + for required in SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS[field]: + evidence[required] = True + if gate in ("G04", "G07", "R-E", "R-L"): + evidence["crash_points"] = ["fixture-before-commit"] + if gate == "G12": + evidence["quota_path_cases"] = list(SCANNER_HEAL_RELEASE_G12_QUOTA_PATH_CASES[field]) + if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS: + evidence["replayed_records"] = 1 + evidence["responsibility_anchor_retained"] = True + evidence["successor_snapshot_published"] = True + if gate == "G14" and field == "ec8_4_evidence": + evidence["topology"] = {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4} + if gate == "G14" and field == "same_window_field_evidence": + evidence["same_window_fields"] = [ + item + for item in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS["G14"] + if item != "same_window_field_evidence" + ] + if gate == "G14" and field == "multi_set_evidence": + evidence["sets"] = 2 + if gate == "G14" and field == "multi_pool_evidence": + evidence["pools"] = 2 + if field == "profile_evidence": + evidence["resolved_samples"] = 1 + artifacts = {} + for artifact_kind in RELEASE_PROFILE_ARTIFACTS: + profile_artifact = artifact_dir / f"{gate}-{field}-{artifact_kind}.json" + write_json(profile_artifact, { + "schema": 1, + "fixture_only": True, + "gate": gate, + "field": field, + "artifact": artifact_kind, + "source_revision": source_revision, + }) + artifacts[artifact_kind] = { + "artifact": profile_artifact.relative_to(bundle_dir).as_posix(), + "sha256": digest(profile_artifact), + "artifact_format": "json", + "source_revision": source_revision, + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + } + evidence["profile_artifacts"] = artifacts + fields[field] = evidence + gates[gate] = { + "status": "pass", + "lane": requirement["lane"], + "evidence_type": "measured", + "evidence_fields": fields, + } + bundle = bundle_dir / "release-evidence.fixture.json" + write_json(bundle, { + "schema": 1, + "fixture_only": True, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + return bundle + + def validate(root: Path) -> list[str]: errors: list[str] = [] errors.extend(check_core_fixtures(root)) @@ -1309,29 +3627,1456 @@ class SelfTests(unittest.TestCase): ) + "" ) - physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation", - "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1], - "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}}, - "inline_data_fingerprint": None} - obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16, - "expected_sha256": "d" * 64, "actual_sha256": "d" * 64, - "expected_physical": physical, "physical": physical} - objects = [dict(obj, key=f"object-{index}") for index in range(9)] - objects[-1] = dict(objects[-1], expected_physical=None) + def oracle_objects(requirement: dict[str, object]) -> list[dict[str, object]]: + topology = requirement["topology"] + total_blocks = topology["nodes"] * topology["drives_per_node"] + erasure_set_drive_count = requirement.get("erasure_set_drive_count", total_blocks) + erasure = requirement.get("erasure") + if erasure is None: + parity_blocks = 4 if erasure_set_drive_count == 12 else erasure_set_drive_count // 2 + data_blocks = erasure_set_drive_count - parity_blocks + else: + data_blocks = erasure["data_blocks"] + parity_blocks = erasure["parity_blocks"] + physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation", + "erasure_index": 1, "data_blocks": data_blocks, "parity_blocks": parity_blocks, + "expected_part_numbers": [1], + "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}}, + "inline_data_fingerprint": None} + obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16, + "expected_sha256": "d" * 64, "actual_sha256": "d" * 64, + "expected_physical": physical, "physical": physical} + count = requirement.get("min_objects", 9) + objects = [dict(obj, key=f"object-{index}") for index in range(count)] + objects[-1] = dict(objects[-1], expected_physical=None) + return objects + for case_id, requirement in requirements.items(): + objects = oracle_objects(requirement) write_json(run_dir / requirement["oracle"], { "schema": 1, "evidence": requirement["evidence"], "case": case_id, "run_id": "a" * 32, "source_revision": "b" * 40, "test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40, "features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""}, "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"], - "topology": requirement["topology"], "pid_before": 10, "pid_after": 11, + "topology": requirement["topology"], + "erasure_set_drive_count": requirement.get( + "erasure_set_drive_count", + requirement["topology"]["nodes"] * requirement["topology"]["drives_per_node"], + ), + "sets": requirement.get("sets", 1), + "pools": requirement.get("pools", 1), + "outage_target_manifest_required": requirement.get("outage_target_manifest_required", True), + "distributed_ec_invalidation": True, + "peer_count": requirement["topology"]["nodes"], + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + "pid_before": 10, "pid_after": 11, "unclean_shutdown_marker": requirement["unclean_shutdown_marker"], - "objects": objects, "node_listings": [[item["key"] for item in objects]] * 4, + "objects": objects, "node_listings": [[item["key"] for item in objects]] * requirement["topology"]["nodes"], }) finish_scanner_heal_receipt(run_dir, 0, root) return root, run_dir + def test_scanner_heal_case_lane_is_required_for_runner_profile(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, _ = self.scanner_heal_fixture(Path(tmp)) + registry_path = root / ".config/scanner-heal-required-tests.json" + registry = read_json(registry_path) + del registry["cases"]["ec84-target-drive-restart"]["lane"] + write_json(registry_path, registry) + with self.assertRaisesRegex(ValueError, "invalid nextest profile lane for ec84-target-drive-restart"): + scanner_heal_oracle_names(root) + + def scanner_heal_release_bundle_fixture(self, directory: Path) -> tuple[Path, Path]: + """Parser fixtures only; the bundle is not runtime evidence.""" + root, _ = self.scanner_heal_fixture(directory) + bundle_dir = directory / "bundle" + artifact_dir = bundle_dir / "artifacts" + artifact_dir.mkdir(parents=True) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + requirements, _, _ = scanner_heal_release_requirements(registry) + gates = {} + source_revision = "b" * 40 + started = datetime(2026, 9, 8, 0, 0, tzinfo=timezone.utc) + for gate, requirement in requirements.items(): + fields = {} + for field in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS[gate]: + artifact = artifact_dir / f"{gate}-{field}.json" + run_id = f"{gate.lower()}-{field.replace('_', '-')}-run" + window_id = f"{gate.lower()}-window" + duration = 60 + evidence = { + "artifact": artifact.relative_to(bundle_dir).as_posix(), + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "started_at": started.isoformat().replace("+00:00", "Z"), + "command": ["cargo", "nextest", "run", requirement["description"]], + "artifact_format": "json", + "summary": f"parser fixture for {gate}.{field}", + } + if gate.startswith("P"): + duration = 900 + evidence["duration_seconds"] = duration + if gate == "P3" and field == "two_hour_pressure_measurement": + duration = 7200 + evidence["duration_seconds"] = duration + if gate == "P4" and field == "mrf_cleanup_gc_soak_evidence": + duration = 7200 + evidence["duration_seconds"] = duration + evidence["finished_at"] = (started + timedelta(seconds=duration)).isoformat().replace("+00:00", "Z") + if gate == "G10" and field == "scheduler_bound_evidence": + evidence.update({ + "max_deferred_items": 256, + "max_deferred_bytes": 262144, + "max_retry_age_seconds": 30, + "duplicate_task_count": 0, + }) + if gate == "G10" and field == "pressure_recovery_evidence": + evidence.update({ + "pressure_pacing_engaged": True, + "recovery_window_seconds": 5, + "lock_hold_p95_ms": 0, + "foreground_latency_p95_ms": 10, + }) + if gate == "P1" and field == "foreground_latency_throughput_measurement": + evidence.update({ + "foreground_latency_p95_ms": 10, + "foreground_latency_p99_ms": 20, + "throughput_ops_per_second": 100, + "error_count": 0, + }) + if gate == "P3" and field == "two_hour_pressure_measurement": + evidence.update({ + "fixed_offered_load": True, + "foreground_latency_p99_ms": 20, + "attempt_cost_samples": 1, + }) + if gate == "P3" and field == "heal_capacity_measurement": + evidence.update({"completed_heal_objects": 1, "duplicate_task_count": 0}) + if gate == "P3" and field == "recovery_window_measurement": + evidence.update({"pressure_recovery_window_seconds": 5, "lock_hold_p95_ms": 0}) + if gate == "G02" and field == "bounded_checkpoint_oracle": + evidence.update({ + "checkpoint_progress_bounded": True, + "raw_entry_budget": 8, + "max_raw_entries_per_round": 8, + "max_objects_processed_per_round": 8, + "bounded_work_quantum_observed": True, + "durable_checkpoint_committed": True, + "no_unbounded_tail": True, + }) + if gate == "G02" and field == "independent_version_inventory": + evidence.update({ + "independent_version_inventory_observed": True, + "objects_expected": 16, + "objects_retained": 16, + "versions_retained": 16, + "bytes_retained": 16, + }) + if gate == "G01" and field == "root_authority_evidence": + evidence.update({ + "root_authority_cases": list(SCANNER_HEAL_RELEASE_G01_ROOT_AUTHORITY_CASES), + "root_cas_observed": True, + "root_readback_observed": True, + "incomplete_root_rejected": True, + "stale_root_rejected": True, + }) + if gate == "G01" and field == "quota_authority_evidence": + evidence.update({ + "quota_authority_cases": list(SCANNER_HEAL_RELEASE_G01_QUOTA_AUTHORITY_CASES), + "quota_floor_readback_observed": True, + "over_limit_put_rejected": True, + "rejected_object_invisible": True, + "quota_fails_closed_without_authority": True, + }) + if gate in ("G03", "G09", "R-L"): + evidence["versions"] = ["a" * 40, source_revision] + evidence["mixed_version_role"] = SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[(gate, field)] + if gate in ("G04", "G07", "R-E", "R-L"): + evidence["crash_points"] = ["before-commit"] + if gate == "G03": + evidence["scoped_ack_cases"] = list(SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES[field]) + for required in SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS[field]: + evidence[required] = True + if gate == "G04" and field == "root_floor_intent_crash_evidence": + evidence["durable_intent_cases"] = list(SCANNER_HEAL_RELEASE_CRASH_BOUNDARY_FIELDS[(gate, field)]) + evidence["persist_failure_blocks_acceptance"] = True + if gate == "G12": + evidence["quota_path_cases"] = list(SCANNER_HEAL_RELEASE_G12_QUOTA_PATH_CASES[field]) + if gate == "G05" and field == "per_object_outcome_oracle": + evidence["per_object_outcome_cases"] = list(SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES) + evidence["outcome_counts"] = {"repaired": 4, "healthy": 3, "skipped": 2, "failed": 1} + evidence["status_matches_object_oracle"] = True + if gate == "G05" and field == "terminal_retention_bounds": + evidence["terminal_retention_cases"] = list(SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES) + evidence["terminal_retention_window_seconds"] = 3600 + evidence["max_terminal_record_age_seconds"] = 3599 + evidence["terminal_records_pruned_after_window"] = 2 + if gate == "G06" and field == "concurrent_status_evidence": + evidence["concurrent_status_cases"] = list(SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES) + evidence["status_samples"] = 4 + evidence["all_status_responses_http_success"] = True + evidence["partial_status_reports_degraded"] = True + if gate == "G06" and field == "legacy_client_compatibility": + evidence["legacy_client_cases"] = list(SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES) + evidence["rustfs_and_minio_paths_compatible"] = True + evidence["empty_body_status_requests_accepted"] = True + if gate == "G06" and field == "truncation_behavior": + evidence["truncation_cases"] = list(SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES) + evidence["truncated_payloads_rejected"] = True + evidence["max_status_payload_bytes"] = 4096 + if gate == "R-E" and field == "fixed_budget_restart_evidence": + evidence.update({ + "fixed_budget_restart_converged": True, + "restart_rounds": 3, + "raw_entry_budget": 8, + "no_unbudgeted_final_sweep": True, + }) + if gate == "R-E" and field == "enumeration_evidence": + evidence.update({ + "raw_enumeration_observed": True, + "durable_raw_page_commit_observed": True, + "raw_page_index_complete": True, + "enumeration_frontier_retained": True, + }) + if gate == "R-E" and field == "classification_evidence": + evidence.update({ + "classification_observed": True, + "objects_processed": 16, + "objects_retained": 16, + "versions_retained": 16, + "bytes_retained": 16, + }) + if gate == "G09": + evidence["mixed_version_cases"] = list(SCANNER_HEAL_RELEASE_MIXED_VERSION_CASES[field]) + if field == "rollback_payload_evidence": + evidence["rollback_payload_replayed"] = True + if (gate, field) in SCANNER_HEAL_RELEASE_MRF_DURABLE_REPLAY_FIELDS: + evidence["replayed_records"] = 2 + evidence["responsibility_anchor_retained"] = True + evidence["successor_snapshot_published"] = True + if gate == "P4" and field == "mrf_cleanup_gc_soak_evidence": + evidence["cleanup_gc_cases"] = list(SCANNER_HEAL_RELEASE_MRF_CLEANUP_GC_SOAK_CASES) + evidence["verified_idle_gc_observed"] = True + evidence["pending_responsibilities_after_gc"] = 0 + evidence["stale_journals_after_gc"] = 0 + if gate == "G07": + case_field = { + "mrf_responsibility_oracle": "mrf_responsibility_cases", + "commit_boundary_crash_matrix": "commit_crash_cases", + }[field] + evidence[case_field] = list(SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES[field]) + if gate == "G08": + case_field = { + "mrf_capacity_evidence": "capacity_cases", + "disk_full_matrix": "disk_full_cases", + "replica_loss_matrix": "replica_loss_cases", + }[field] + evidence[case_field] = list(SCANNER_HEAL_RELEASE_G08_REQUIRED_CASES[field]) + if gate == "G10" and field == "scheduler_bound_evidence": + evidence["scheduler_bounds"] = list(SCANNER_HEAL_RELEASE_SCHEDULER_BOUNDS) + evidence["duplicate_task_bound_observed"] = True + if gate == "G10" and field == "pressure_recovery_evidence": + evidence["pressure_metrics"] = { + "foreground_p95_ms": 40.0, + "foreground_p99_ms": 90.0, + "throughput_ops": 250.0, + "error_rate": 0.0, + "heal_lock_wait_p99_ms": 25.0, + "attempt_cost_samples": 8.0, + "foreground_pressure_samples": 120, + "foreground_pressure_high_samples": 12, + } + if gate == "G11" and field == "maintenance_producer_matrix": + evidence["producer_identities"] = list(SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES) + evidence["producer_families"] = list(SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES) + evidence["matrix_cases"] = list(SCANNER_HEAL_RELEASE_G11_REQUIRED_CASES[field]) + evidence["durable_identity_observed"] = True + evidence["durable_journal_replay_observed"] = True + evidence["durable_journal_hydration_bound"] = True + evidence["generation_window_bound"] = True + evidence["restart_gap_absent"] = True + evidence["overflow_absent"] = True + evidence["ack_clear_generation_observed"] = True + evidence["invalid_journal_inputs_rejected"] = list(SCANNER_HEAL_DURABLE_JOURNAL_FAIL_CLOSED_CASES) + if gate == "G11" and field == "complete_producer_inventory": + evidence["required_producer_identities"] = list(SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES) + evidence["observed_producer_identities"] = list(SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES) + evidence["required_producer_families"] = list(SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES) + evidence["observed_producer_families"] = list(SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES) + evidence["missing_producer_identities"] = [] + evidence["unknown_producer_excluded"] = True + if gate == "G13" and field == "quorum_minus_one_matrix": + evidence["quorum_cases"] = list(SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES[field]) + evidence["no_success_at_quorum_minus_one"] = True + evidence["exact_quorum_restored"] = True + if gate == "G13" and field == "unknown_disk_remount_matrix": + evidence["remount_cases"] = list(SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES[field]) + evidence["unknown_disks_excluded"] = True + evidence["remounted_disks_revalidated"] = True + evidence["stale_incarnation_rejected"] = True + if gate == "G13" and field == "object_lock_dry_run_grace_evidence": + evidence["grace_cases"] = list(SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES[field]) + evidence["object_lock_denials_preserved"] = True + evidence["dry_run_mutation_count"] = 0 + evidence["grace_outcomes_retained"] = True + if gate == "G14" and field == "ec8_4_evidence": + evidence["topology"] = {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4} + if gate == "G14" and field == "same_window_field_evidence": + evidence["same_window_fields"] = [ + item + for item in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS["G14"] + if item != "same_window_field_evidence" + ] + if gate == "G14" and field == "multi_set_evidence": + evidence["sets"] = 2 + if gate == "G14" and field == "multi_pool_evidence": + evidence["pools"] = 2 + if gate == "G14" and field == "distributed_segment_invalidation_evidence": + evidence["invalidation_domain"] = "distributed-ec" + evidence["distributed_ec_invalidation"] = True + evidence["peer_count"] = 3 + evidence["same_window_remote_proof"] = True + evidence["all_peers_bound_to_generation_window"] = True + if gate == "G14": + case_id = "ec84-multiset-multipool" + case_artifact = artifact_dir / "cases" / f"{gate}-{field}-{case_id}.json" + case_artifact.parent.mkdir(parents=True, exist_ok=True) + write_json(case_artifact, { + "schema": 1, + "case": case_id, + "evidence_type": "measured", + "source_revision": source_revision, + "measurement_window_id": window_id, + "topology": {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4}, + "sets": 2, + "pools": 2, + "invalidation_domain": "distributed-ec", + "distributed_ec_invalidation": True, + "peer_count": 3, + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + }) + evidence["case_evidence"] = [ + { + "case": case_id, + "artifact": case_artifact.relative_to(bundle_dir).as_posix(), + "sha256": digest(case_artifact), + "source_revision": source_revision, + "measurement_window_id": window_id, + }, + ] + if field == "segment_activation_preflight": + evidence["production_activation"] = True + evidence["scanner_segment_reuse_activated"] = True + evidence["proof_inputs"] = list(SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS) + evidence["fail_closed_checks"] = list(SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS) + if field == "cold_segment_reuse_measurement": + evidence["hot_walked_segments"] = 2 + evidence["cold_walked_segments"] = 0 + evidence["full_walk_oracle_equivalent"] = True + evidence["published_root_equivalent"] = True + if field == "post_stop_convergence_measurement": + evidence["writes_stopped"] = True + evidence["last_mutation_observed"] = True + evidence["first_complete_publication"] = True + evidence["post_stop_samples"] = 2 + evidence["post_stop_work_multiple"] = 1.1 + evidence["post_stop_work_multiple_limit"] = 1.2 + evidence["post_stop_work_multiples"] = [1.0, 1.1] + if gate == "P4" and field == "retained_responsibility_evidence": + evidence["duration_seconds"] = 7200 + evidence["finished_at"] = (started + timedelta(seconds=7200)).isoformat().replace("+00:00", "Z") + evidence["retained_responsibility_cases"] = list( + SCANNER_HEAL_RELEASE_P4_RETAINED_RESPONSIBILITY_CASES + ) + evidence["retention_window_seconds"] = 7200 + evidence["idle_cleanup_observed"] = True + evidence["verified_proof_discharge_observed"] = True + if gate == "R-D" and field == "manager_disposition_evidence": + evidence["manager_disposition_cases"] = list(SCANNER_HEAL_RELEASE_RD_MANAGER_CASES) + evidence["manager_dispositions_are_terminal"] = True + if gate == "R-D" and field == "event_disposition_evidence": + evidence["event_disposition_cases"] = list(SCANNER_HEAL_RELEASE_RD_EVENT_CASES) + evidence["events_correlate_to_manager_dispositions"] = True + if gate == "R-D" and field == "ledger_disposition_evidence": + evidence["ledger_disposition_cases"] = list(SCANNER_HEAL_RELEASE_RD_LEDGER_CASES) + evidence["ledger_correlates_to_events"] = True + evidence["ledger_replay_preserves_terminal_disposition"] = True + if gate == "R-D" and field == "grace_handling": + evidence["grace_cases"] = list(SCANNER_HEAL_RELEASE_RD_GRACE_CASES) + evidence["grace_window_seconds"] = 300 + evidence["grace_retention_observed"] = True + evidence["grace_expiry_pruned_terminal_records"] = True + if gate == "R-L" and field == "legacy_source_conflict_evidence": + evidence["legacy_source_conflict_cases"] = list(SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES[field]) + evidence["source_conflicts_rejected"] = True + evidence["takeover_identity_bound"] = True + evidence["legacy_checksum_gap_rejected"] = True + if gate == "R-L" and field == "migration_gap_evidence": + evidence["migration_gap_cases"] = list(SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES[field]) + evidence["migration_gap_closed"] = True + evidence["legacy_sources_fail_closed"] = True + evidence["prior_responsibilities_inherited"] = True + if gate == "R-L" and field == "crash_safe_source_retirement_evidence": + evidence["source_retirement_cases"] = list(SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES[field]) + evidence["source_retirement_is_crash_safe"] = True + evidence["old_source_retained_until_successor"] = True + evidence["recovered_pending_migration"] = True + if field == "profile_evidence": + profile_metrics = { + "resolved_samples": 1, + "allocation_bytes": 1024, + "rss_peak_bytes": 4096, + "save_operations": 2, + "saved_bytes": 2048, + } + evidence.update(profile_metrics) + artifacts = {} + for artifact_kind in RELEASE_PROFILE_ARTIFACTS: + raw_artifact = artifact_dir / f"{gate}-{field}-{artifact_kind}.raw" + raw_artifact.write_text(f"{artifact_kind} measured profile sample\n", encoding="utf-8") + profile_artifact = artifact_dir / f"{gate}-{field}-{artifact_kind}.json" + write_json(profile_artifact, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "gate": gate, + "field": field, + "artifact_kind": artifact_kind, + "raw_profile_artifact": raw_artifact.relative_to(bundle_dir).as_posix(), + "raw_profile_sha256": digest(raw_artifact), + "raw_profile_bytes": raw_artifact.stat().st_size, + "raw_profile_format": "raw", + **{ + metric: profile_metrics[metric] + for metric in PROFILE_ARTIFACT_REQUIRED_METRICS[artifact_kind] + }, + }) + artifacts[artifact_kind] = { + "artifact": profile_artifact.relative_to(bundle_dir).as_posix(), + "sha256": digest(profile_artifact), + "artifact_format": "json", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + } + evidence["profile_artifacts"] = artifacts + if gate == "P1" and field == "cold_walk_share_measurement": + evidence["cold_walk_share"] = 0.5 + evidence["walk_objects"] = 100 + evidence["cold_walk_objects"] = 50 + if gate == "P1" and field == "foreground_latency_throughput_measurement": + evidence["foreground_p95_ms"] = 50.0 + evidence["foreground_p99_ms"] = 100.0 + evidence["throughput_ops"] = 200.0 + evidence["error_rate"] = 0.0 + if gate == "P3" and field == "two_hour_pressure_measurement": + evidence["abba_legs"] = ["A1", "B1", "B2", "A2"] + evidence["scenarios"] = list(SCENARIOS) + evidence["foreground_p95_ms"] = 50.0 + evidence["foreground_p99_ms"] = 100.0 + evidence["throughput_ops"] = 200.0 + if gate == "P3" and field == "heal_capacity_measurement": + evidence["heal_capacity"] = { + "objects": 100, + "versions": 100, + "bytes": 1048576, + "completed_objects": 100, + } + if gate == "P3" and field == "recovery_window_measurement": + evidence["fault_modes"] = ["process-restart", "process-crash-restart"] + evidence["recovery_p95_ms"] = 500.0 + evidence["recovery_p99_ms"] = 1000.0 + artifact_payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "gate": gate, + "field": field, + } + mrf_artifact_kind = SCANNER_HEAL_RELEASE_MRF_ARTIFACT_KINDS.get((gate, field)) + if mrf_artifact_kind is not None: + artifact_payload["artifact_kind"] = mrf_artifact_kind + for mirror_field in release_bundle_json_artifact_mirrored_fields(gate, field): + artifact_payload[mirror_field] = evidence[mirror_field] + write_json(artifact, artifact_payload) + evidence["sha256"] = digest(artifact) + fields[field] = evidence + gates[gate] = { + "status": "pass", + "lane": requirement["lane"], + "evidence_type": "measured", + "evidence_fields": fields, + } + bundle = bundle_dir / "release-evidence.json" + write_json(bundle, {"schema": 1, "evidence": "measured", "source_revision": source_revision, "gates": gates}) + return root, bundle + + def test_scanner_heal_release_bundle_accepts_complete_measured_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "approved") + self.assertTrue(status["release_approved"]) + self.assertEqual(len(status["verified_gates"]), len(SCANNER_HEAL_RELEASE_REQUIRED_GATES)) + self.assertEqual(status["pending_gates"], []) + self.assertEqual(status["pending_lanes"], []) + + def test_scanner_heal_release_bundle_assembler_copies_measured_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, descriptor = self.scanner_heal_release_bundle_fixture(Path(tmp)) + with mock.patch("subprocess.check_output", return_value="b" * 40): + bundle, status = assemble_scanner_heal_release_bundle(root, descriptor, Path(tmp) / "assembled") + self.assertEqual(status["decision"], "approved") + self.assertTrue(status["release_approved"]) + assembled = read_json(bundle) + evidence = assembled["gates"]["G01"]["evidence_fields"]["root_authority_evidence"] + self.assertEqual(evidence["artifact"], "artifacts/G01-root_authority_evidence.json") + self.assertEqual(evidence["sha256"], digest(bundle.parent / evidence["artifact"])) + self.assertTrue((bundle.parent / evidence["artifact"]).is_file()) + profile = assembled["gates"]["P1"]["evidence_fields"]["profile_evidence"] + profile_artifact = profile["profile_artifacts"]["save-frequency"] + profile_wrapper = bundle.parent / profile_artifact["artifact"] + profile_payload = read_json(profile_wrapper) + raw_profile = bundle.parent / profile_payload["raw_profile_artifact"] + self.assertTrue(raw_profile.is_file()) + self.assertEqual(profile_payload["raw_profile_sha256"], digest(raw_profile)) + self.assertEqual(profile_artifact["sha256"], digest(profile_wrapper)) + + def test_scanner_heal_release_bundle_assembler_merges_measured_descriptors(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, descriptor = self.scanner_heal_release_bundle_fixture(Path(tmp)) + source = read_json(descriptor) + descriptor_paths = [] + for gate in ("G01", "G09"): + partial = descriptor.parent / f"release-bundle-{gate}.json" + write_json(partial, { + "schema": 1, + "evidence": "measured", + "source_revision": source["source_revision"], + "gates": {gate: source["gates"][gate]}, + }) + descriptor_paths.append(partial) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + bundle, status = assemble_scanner_heal_release_bundle_descriptors( + root, + descriptor_paths, + Path(tmp) / "assembled", + ) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertIn("G01", status["verified_gates"]) + self.assertIn("G09", status["verified_gates"]) + self.assertIn("G02", status["pending_gates"]) + assembled = read_json(bundle) + self.assertEqual(sorted(assembled["gates"]), ["G01", "G09"]) + g09 = assembled["gates"]["G09"]["evidence_fields"]["mixed_version_reader_evidence"] + self.assertTrue((bundle.parent / g09["artifact"]).is_file()) + + def test_scanner_heal_release_bundle_assembler_rejects_duplicate_descriptor_gate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, descriptor = self.scanner_heal_release_bundle_fixture(Path(tmp)) + with mock.patch("subprocess.check_output", return_value="b" * 40): + with self.assertRaisesRegex(ValueError, "repeats gate: G01"): + assemble_scanner_heal_release_bundle_descriptors( + root, + [descriptor, descriptor], + Path(tmp) / "assembled", + ) + + def test_scanner_heal_release_bundle_assembler_rejects_mixed_descriptor_revisions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, descriptor = self.scanner_heal_release_bundle_fixture(Path(tmp)) + source = read_json(descriptor) + g01 = descriptor.parent / "release-bundle-g01.json" + g09 = descriptor.parent / "release-bundle-g09.json" + write_json(g01, { + "schema": 1, + "evidence": "measured", + "source_revision": source["source_revision"], + "gates": {"G01": source["gates"]["G01"]}, + }) + write_json(g09, { + "schema": 1, + "evidence": "measured", + "source_revision": "c" * 40, + "gates": {"G09": source["gates"]["G09"]}, + }) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + with self.assertRaisesRegex(ValueError, "source revisions differ"): + assemble_scanner_heal_release_bundle_descriptors(root, [g01, g09], Path(tmp) / "assembled") + + def test_scanner_heal_release_bundle_gate_accepts_single_measured_gate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + data["gates"] = {"G09": data["gates"]["G09"]} + write_json(bundle, data) + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_gate_status(root, bundle, "G09") + full_status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "verified") + self.assertFalse(status["release_approved"]) + self.assertEqual(status["verified_gate"], "G09") + self.assertIn("G07", status["pending_gates"]) + self.assertEqual(full_status["decision"], "blocked") + self.assertFalse(full_status["release_approved"]) + + def test_scanner_heal_release_bundle_gate_rejects_missing_field(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + del data["gates"]["G09"]["evidence_fields"]["rollback_payload_evidence"] + write_json(bundle, data) + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_gate_status(root, bundle, "G09") + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertIn("missing required fields: rollback_payload_evidence", status["rejected_errors"]) + + def test_scanner_heal_release_bundle_gate_rejects_unknown_gate(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + data["gates"] = {"G09": data["gates"]["G09"], "Z99": {"status": "pass"}} + write_json(bundle, data) + with mock.patch("subprocess.check_output", return_value="b" * 40): + with self.assertRaisesRegex(ValueError, "unknown gates: Z99"): + scanner_heal_release_bundle_gate_status(root, bundle, "G09") + + def test_scanner_heal_release_bundle_assembler_rejects_fixture_descriptor(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, _ = self.scanner_heal_fixture(Path(tmp)) + with mock.patch("subprocess.check_output", return_value="b" * 40): + descriptor = write_scanner_heal_release_bundle_fixture(root, Path(tmp) / "bundle") + with self.assertRaisesRegex(ValueError, "fixture_only"): + assemble_scanner_heal_release_bundle(root, descriptor, Path(tmp) / "assembled") + + def test_scanner_heal_release_bundle_assembler_rejects_hash_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, descriptor = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(descriptor) + data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]["sha256"] = "0" * 64 + write_json(descriptor, data) + + with self.assertRaisesRegex(ValueError, "descriptor artifact hash mismatch"): + assemble_scanner_heal_release_bundle(root, descriptor, Path(tmp) / "assembled") + + def test_scanner_heal_release_bundle_fixture_writer_is_not_release_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, _ = self.scanner_heal_fixture(Path(tmp)) + with mock.patch("subprocess.check_output", return_value="b" * 40): + bundle = write_scanner_heal_release_bundle_fixture(root, Path(tmp) / "bundle") + with self.assertRaisesRegex(ValueError, "fixture_only"): + scanner_heal_release_bundle_status(root, bundle) + data = read_json(bundle) + self.assertTrue(data["fixture_only"]) + self.assertEqual(data["source_revision"], "b" * 40) + + def test_scanner_heal_release_bundle_rejects_synthetic_or_missing_fields(self) -> None: + for fault in ("synthetic", "missing-field", "hash", "empty-artifact"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + if fault == "synthetic": + data["evidence"] = "synthetic" + elif fault == "missing-field": + del data["gates"]["G09"]["evidence_fields"]["rollback_payload_evidence"] + elif fault == "hash": + artifact = bundle.parent / data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]["artifact"] + artifact.write_text(artifact.read_text(encoding="utf-8") + "\n", encoding="utf-8") + else: + artifact = bundle.parent / data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]["artifact"] + artifact.write_text("", encoding="utf-8") + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + if fault == "synthetic": + with self.assertRaisesRegex(ValueError, "must be measured"): + scanner_heal_release_bundle_status(root, bundle) + else: + status = scanner_heal_release_bundle_status(root, bundle) + if fault != "synthetic": + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertTrue(status["rejected_gates"]) + + def test_scanner_heal_release_bundle_enforces_topology_duration_profile_and_versions(self) -> None: + for fault, gate, field, mutation, expected in ( + ("topology", "G14", "ec8_4_evidence", lambda item: item.update({"topology": {"erasure": "EC4+2", "nodes": 2, "drives_per_node": 3}}), "JSON artifact topology mismatch"), + ("missing-duration", "P1", "cold_walk_share_measurement", lambda item: item.pop("duration_seconds"), "duration_seconds"), + ("duration", "P3", "two_hour_pressure_measurement", lambda item: item.update({"duration_seconds": 7199}), "two hours"), + ("mrf-cleanup-soak-duration", "P4", "mrf_cleanup_gc_soak_evidence", lambda item: item.update({"duration_seconds": 7199}), "two hours"), + ("profile", "P1", "profile_evidence", lambda item: item.pop("resolved_samples"), "resolved_samples"), + ( + "profile-artifact", + "P1", + "profile_evidence", + lambda item: item["profile_artifacts"].pop("flamegraph"), + "missing profile artifacts", + ), + ( + "profile-artifact-hash", + "P1", + "profile_evidence", + lambda item: item["profile_artifacts"]["rss-samples"].update({"sha256": "0" * 64}), + "artifact hash mismatch", + ), + ( + "profile-artifact-source", + "P1", + "profile_evidence", + lambda item: item["profile_artifacts"]["allocation-profile"].update({"source_revision": "c" * 40}), + "source revision mismatch", + ), + ( + "profile-artifact-run", + "P1", + "profile_evidence", + lambda item: item["profile_artifacts"]["flamegraph"].update({"run_id": "p1-different-profile-run"}), + "run_id mismatch", + ), + ( + "profile-artifact-window", + "P1", + "profile_evidence", + lambda item: item["profile_artifacts"]["save-frequency"].update( + {"measurement_window_id": "p1-different-window"} + ), + "measurement window mismatch", + ), + ("g05-outcome-cases", "G05", "per_object_outcome_oracle", lambda item: item["per_object_outcome_cases"].remove("object-failed-and-retained"), "missing cases"), + ("g05-outcome-counts", "G05", "per_object_outcome_oracle", lambda item: item["outcome_counts"].pop("failed"), "outcome_counts.failed"), + ( + "g05-retention-window", + "G05", + "terminal_retention_bounds", + lambda item: item.update({"max_terminal_record_age_seconds": item["terminal_retention_window_seconds"] + 1}), + "exceeds retention window", + ), + ( + "g06-concurrent-status", + "G06", + "concurrent_status_evidence", + lambda item: item["concurrent_status_cases"].remove("status-while-peer-down"), + "missing cases", + ), + ( + "g06-legacy-client", + "G06", + "legacy_client_compatibility", + lambda item: item.update({"empty_body_status_requests_accepted": False}), + "empty_body_status_requests_accepted", + ), + ( + "g06-truncation", + "G06", + "truncation_behavior", + lambda item: item["truncation_cases"].remove("truncated-node-status-reject"), + "missing cases", + ), + ("versions", "G09", "mixed_version_reader_evidence", lambda item: item.update({"versions": [1, 2]}), "mixed-version"), + ("stale-versions", "G09", "mixed_version_writer_evidence", lambda item: item.update({"versions": ["a" * 40, "c" * 40]}), "tested source revision"), + ("g08-capacity-cases", "G08", "mrf_capacity_evidence", lambda item: item.update({"capacity_cases": ["queue-count-limit"]}), "missing cases"), + ( + "g08-disk-full-cases", + "G08", + "disk_full_matrix", + lambda item: item["disk_full_cases"].remove("cleanup-delete-enospc"), + "missing cases", + ), + ("g08-replica-loss-cases", "G08", "replica_loss_matrix", lambda item: item.update({"replica_loss_cases": ["single-replica-loss"]}), "missing cases"), + ("g07-responsibility-cases", "G07", "mrf_responsibility_oracle", lambda item: item.update({"mrf_responsibility_cases": ["legacy-journal-replay"]}), "missing cases"), + ("g07-crash-cases", "G07", "commit_boundary_crash_matrix", lambda item: item.pop("commit_crash_cases"), "non-empty string list"), + ("mrf-records", "G07", "mrf_responsibility_oracle", lambda item: item.pop("replayed_records"), "replayed_records"), + ("mrf-anchor", "G07", "commit_boundary_crash_matrix", lambda item: item.update({"responsibility_anchor_retained": False}), "retained MRF responsibility anchors"), + ("mrf-successor", "P4", "retained_responsibility_evidence", lambda item: item.pop("successor_snapshot_published"), "successor snapshot"), + ( + "g02-raw-budget", + "G02", + "bounded_checkpoint_oracle", + lambda item: item.update({"max_raw_entries_per_round": item["raw_entry_budget"] + 1}), + "raw entries exceed fixed budget", + ), + ( + "g02-durable-checkpoint", + "G02", + "bounded_checkpoint_oracle", + lambda item: item.update({"durable_checkpoint_committed": False}), + "durable_checkpoint_committed", + ), + ( + "g02-version-inventory", + "G02", + "independent_version_inventory", + lambda item: item.update({"versions_retained": item["objects_retained"] - 1}), + "version inventory", + ), + ( + "re-converged", + "R-E", + "fixed_budget_restart_evidence", + lambda item: item.update({"fixed_budget_restart_converged": False}), + "fixed_budget_restart_converged", + ), + ( + "re-enumeration-commit", + "R-E", + "enumeration_evidence", + lambda item: item.update({"durable_raw_page_commit_observed": False}), + "durable_raw_page_commit_observed", + ), + ( + "re-classification-inventory", + "R-E", + "classification_evidence", + lambda item: item.update({"bytes_retained": item["objects_retained"] - 1}), + "byte inventory", + ), + ( + "mrf-cleanup-gc-cases", + "P4", + "mrf_cleanup_gc_soak_evidence", + lambda item: item["cleanup_gc_cases"].remove("repeated-replay-no-resurrection"), + "cleanup_gc_cases missing cases", + ), + ( + "mrf-cleanup-gc-idle", + "P4", + "mrf_cleanup_gc_soak_evidence", + lambda item: item.update({"verified_idle_gc_observed": False}), + "verified idle GC", + ), + ( + "mrf-cleanup-gc-pending", + "P4", + "mrf_cleanup_gc_soak_evidence", + lambda item: item.update({"pending_responsibilities_after_gc": 1}), + "zero pending responsibilities", + ), + ( + "mrf-cleanup-gc-stale", + "P4", + "mrf_cleanup_gc_soak_evidence", + lambda item: item.update({"stale_journals_after_gc": 1}), + "zero stale journals", + ), + ("same-window-fields", "G14", "same_window_field_evidence", lambda item: item.update({"same_window_fields": ["ec8_4_evidence", "multi_set_evidence"]}), "JSON artifact same_window_fields mismatch"), + ("activation-disabled", "G11", "segment_activation_preflight", lambda item: item.update({"production_activation": False}), "production activation is enabled"), + ( + "activation-runtime-disabled", + "G11", + "segment_activation_preflight", + lambda item: item.update({"scanner_segment_reuse_activated": False}), + "runtime activation gate is enabled", + ), + ( + "activation-missing-fail-closed", + "G11", + "segment_activation_preflight", + lambda item: item["fail_closed_checks"].remove("overflow"), + "missing G11.segment_activation_preflight.fail_closed_checks", + ), + ( + "activation-missing-proof-input", + "G11", + "segment_activation_preflight", + lambda item: item["proof_inputs"].remove("process_epoch"), + "missing G11.segment_activation_preflight.proof_inputs", + ), + ( + "activation-missing-durable-journal-input", + "G11", + "segment_activation_preflight", + lambda item: item["proof_inputs"].remove("durable_dirty_producer_journal"), + "missing G11.segment_activation_preflight.proof_inputs", + ), + ( + "activation-missing-ack-guard", + "G11", + "segment_activation_preflight", + lambda item: item["fail_closed_checks"].remove("stale_ack_generation"), + "missing G11.segment_activation_preflight.fail_closed_checks", + ), + ( + "producer-matrix-missing", + "G11", + "maintenance_producer_matrix", + lambda item: item["producer_identities"].remove("tier_expiration"), + "producer_identities missing cases", + ), + ( + "producer-matrix-no-journal-replay", + "G11", + "maintenance_producer_matrix", + lambda item: item.update({"durable_journal_replay_observed": False}), + "durable_journal_replay_observed", + ), + ( + "producer-matrix-no-hydration-bound", + "G11", + "maintenance_producer_matrix", + lambda item: item.update({"durable_journal_hydration_bound": False}), + "durable_journal_hydration_bound", + ), + ( + "producer-matrix-no-ack-clear-generation", + "G11", + "maintenance_producer_matrix", + lambda item: item.update({"ack_clear_generation_observed": False}), + "ack_clear_generation_observed", + ), + ( + "producer-matrix-missing-invalid-journal-case", + "G11", + "maintenance_producer_matrix", + lambda item: item["invalid_journal_inputs_rejected"].remove("mixed-version-producer"), + "invalid_journal_inputs_rejected missing cases", + ), + ( + "producer-inventory-missing", + "G11", + "complete_producer_inventory", + lambda item: item.update({"missing_producer_identities": ["tier_expiration"]}), + "zero missing producer identities", + ), + ( + "producer-inventory-unknown", + "G11", + "complete_producer_inventory", + lambda item: item.update({"unknown_producer_excluded": False}), + "unknown_producer_excluded", + ), + ( + "quorum-minus-one-success", + "G13", + "quorum_minus_one_matrix", + lambda item: item.update({"no_success_at_quorum_minus_one": False}), + "no_success_at_quorum_minus_one", + ), + ( + "remount-revalidation", + "G13", + "unknown_disk_remount_matrix", + lambda item: item.update({"remounted_disks_revalidated": False}), + "remounted_disks_revalidated", + ), + ( + "dry-run-mutates", + "G13", + "object_lock_dry_run_grace_evidence", + lambda item: item.update({"dry_run_mutation_count": 1}), + "dry_run_mutation_count", + ), + ( + "distributed-invalidation", + "G14", + "distributed_segment_invalidation_evidence", + lambda item: item.update({"distributed_ec_invalidation": False}), + "JSON artifact distributed_ec_invalidation mismatch", + ), + ( + "distributed-peer-window", + "G14", + "distributed_segment_invalidation_evidence", + lambda item: item.update({"all_peers_bound_to_generation_window": False}), + "JSON artifact all_peers_bound_to_generation_window mismatch", + ), + ( + "post-stop-work-multiple", + "P2", + "post_stop_convergence_measurement", + lambda item: item.update({"post_stop_work_multiple": 1.3}), + "post-stop work multiple limit", + ), + ( + "post-stop-publication", + "P2", + "post_stop_convergence_measurement", + lambda item: item.update({"first_complete_publication": False}), + "first complete publication", + ), + ( + "cold-segment-walk", + "P2", + "cold_segment_reuse_measurement", + lambda item: item.update({"cold_walked_segments": 1}), + "cold_walked_segments", + ), + ( + "cold-segment-oracle", + "P2", + "cold_segment_reuse_measurement", + lambda item: item.update({"full_walk_oracle_equivalent": False}), + "full-walk oracle equivalence", + ), + ( + "p4-retained-responsibility-cases", + "P4", + "retained_responsibility_evidence", + lambda item: item["retained_responsibility_cases"].remove("idle-cleanup-reclaims-replay-source"), + "retained_responsibility_cases missing cases", + ), + ( + "p4-retention-window", + "P4", + "retained_responsibility_evidence", + lambda item: item.update({"retention_window_seconds": 7199}), + "retention_window_seconds", + ), + ( + "p4-idle-cleanup", + "P4", + "retained_responsibility_evidence", + lambda item: item.update({"idle_cleanup_observed": False}), + "idle_cleanup_observed", + ), + ( + "p4-proof-discharge", + "P4", + "retained_responsibility_evidence", + lambda item: item.pop("verified_proof_discharge_observed"), + "verified_proof_discharge_observed", + ), + ("scheduler-duplicates", "G10", "scheduler_bound_evidence", lambda item: item.pop("duplicate_task_count"), "duplicate_task_count"), + ("p1-throughput", "P1", "foreground_latency_throughput_measurement", lambda item: item.pop("throughput_ops_per_second"), "throughput_ops_per_second"), + ("p3-fixed-load", "P3", "two_hour_pressure_measurement", lambda item: item.update({"fixed_offered_load": False}), "fixed offered load"), + ( + "g01-root-cases", + "G01", + "root_authority_evidence", + lambda item: item["root_authority_cases"].remove("stale-root-rejected"), + "root_authority_cases missing cases", + ), + ( + "g01-quota-fail-closed", + "G01", + "quota_authority_evidence", + lambda item: item.update({"quota_fails_closed_without_authority": False}), + "quota_fails_closed_without_authority", + ), + ( + "rd-manager", + "R-D", + "manager_disposition_evidence", + lambda item: item["manager_disposition_cases"].remove("rejected-policy"), + "missing cases", + ), + ( + "rd-event", + "R-D", + "event_disposition_evidence", + lambda item: item.update({"events_correlate_to_manager_dispositions": False}), + "events_correlate_to_manager_dispositions", + ), + ( + "rd-ledger", + "R-D", + "ledger_disposition_evidence", + lambda item: item.update({"ledger_replay_preserves_terminal_disposition": False}), + "ledger_replay_preserves_terminal_disposition", + ), + ( + "rd-grace", + "R-D", + "grace_handling", + lambda item: item["grace_cases"].remove("restart-preserves-grace-clock"), + "missing cases", + ), + ): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + mutation(data["gates"][gate]["evidence_fields"][field]) + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertTrue(any(expected in error for error in status["rejected_gates"][gate])) + + def test_scanner_heal_release_bundle_requires_mixed_version_field_roles(self) -> None: + for gate, field, wrong_role in ( + ("G03", "mixed_peer_ack_fallback_oracle", "mixed-version-reader"), + ("G09", "mixed_version_reader_evidence", "mixed-version-writer"), + ("G09", "mixed_version_writer_evidence", "mixed-version-reader"), + ("G09", "rollback_payload_evidence", "mixed-version-reader"), + ("R-L", "crash_safe_source_retirement_evidence", "migration-gap"), + ): + with self.subTest(gate=gate, field=field), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + data["gates"][gate]["evidence_fields"][field]["mixed_version_role"] = wrong_role + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertTrue(any("mixed-version role" in error for error in status["rejected_gates"][gate])) + + def test_scanner_heal_release_bundle_requires_field_provenance(self) -> None: + for fault, mutation, expected in ( + ("source", lambda item: item.update({"source_revision": "c" * 40}), "source revision mismatch"), + ("run-id", lambda item: item.pop("run_id"), "run_id"), + ("window-id", lambda item: item.update({"measurement_window_id": item["run_id"]}), "separate run"), + ("started-at", lambda item: item.update({"started_at": "not-a-time"}), "timestamp"), + ("finished-at", lambda item: item.update({"finished_at": "2026-09-07T00:00:00Z"}), "timestamps are inverted"), + ("command", lambda item: item.update({"command": []}), "command provenance"), + ("artifact-format", lambda item: item.pop("artifact_format"), "artifact_format"), + ): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + mutation(data["gates"]["G01"]["evidence_fields"]["root_authority_evidence"]) + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertTrue(any(expected in error for error in status["rejected_gates"]["G01"]), fault) + + def test_scanner_heal_release_bundle_requires_json_artifact_provenance(self) -> None: + for fault, mutation, evidence_path, expected in ( + ( + "summary-window", + lambda payload: payload.update({"measurement_window_id": "p1-stale-window"}), + ("P1", "profile_evidence"), + "JSON artifact measurement window mismatch", + ), + ( + "summary-source", + lambda payload: payload.update({"source_revision": "c" * 40}), + ("G09", "mixed_version_reader_evidence"), + "JSON artifact source revision mismatch", + ), + ( + "profile-kind", + lambda payload: payload.update({"artifact_kind": "flamegraph"}), + ("P1", "profile_evidence", "allocation-profile"), + "JSON artifact kind mismatch", + ), + ( + "profile-run", + lambda payload: payload.update({"run_id": "p1-different-profile-run"}), + ("P1", "profile_evidence", "rss-samples"), + "JSON artifact run_id mismatch", + ), + ( + "profile-raw-hash", + lambda payload: payload.update({"raw_profile_sha256": "0" * 64}), + ("P1", "profile_evidence", "flamegraph"), + "raw profile hash mismatch", + ), + ( + "profile-save-cost", + lambda payload: payload.pop("saved_bytes"), + ("P1", "profile_evidence", "save-frequency"), + "save-frequency.saved_bytes", + ), + ( + "fixture-marker", + lambda payload: payload.update({"fixture": True}), + ("G08", "disk_full_matrix"), + "JSON artifact is fixture", + ), + ( + "g05-outcome-mirror", + lambda payload: payload["per_object_outcome_cases"].remove("object-skipped-by-policy"), + ("G05", "per_object_outcome_oracle"), + "per_object_outcome_cases missing cases", + ), + ( + "g01-authority-mirror", + lambda payload: payload["root_authority_cases"].remove("incomplete-root-rejected"), + ("G01", "root_authority_evidence"), + "root_authority_cases missing cases", + ), + ( + "g06-truncation-mirror", + lambda payload: payload["truncation_cases"].remove("trailing-data-node-status-reject"), + ("G06", "truncation_behavior"), + "truncation_cases missing cases", + ), + ( + "mrf-artifact-kind", + lambda payload: payload.update({"artifact_kind": "generic-json"}), + ("G08", "disk_full_matrix"), + "JSON artifact kind must be mrf-disk-full-enospc-matrix", + ), + ( + "g08-case-mirror", + lambda payload: payload["disk_full_cases"].remove("manifest-write-enospc"), + ("G08", "disk_full_matrix"), + "JSON artifact missing cases", + ), + ( + "p4-gc-mirror", + lambda payload: payload.update({"pending_responsibilities_after_gc": 1}), + ("P4", "mrf_cleanup_gc_soak_evidence"), + "JSON artifact requires zero pending responsibilities", + ), + ( + "rd-grace-mirror", + lambda payload: payload["grace_cases"].remove("grace-expired-prunes-terminal"), + ("R-D", "grace_handling"), + "grace_cases missing cases", + ), + ( + "rl-migration-gap-mirror", + lambda payload: payload.update({"migration_gap_closed": False}), + ("R-L", "migration_gap_evidence"), + "migration_gap_closed", + ), + ( + "g14-topology-mirror", + lambda payload: payload.update({"topology": {"erasure": "EC8+4", "nodes": 4, "drives_per_node": 4}}), + ("G14", "ec8_4_evidence"), + "JSON artifact topology mismatch", + ), + ( + "g14-case-source", + lambda payload: payload["case_evidence"][0].update({"source_revision": "c" * 40}), + ("G14", "multi_pool_evidence"), + "JSON artifact case_evidence mismatch", + ), + ): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + gate, field, *artifact_kind = evidence_path + evidence = data["gates"][gate]["evidence_fields"][field] + if artifact_kind: + item = evidence["profile_artifacts"][artifact_kind[0]] + else: + item = evidence + artifact = bundle.parent / item["artifact"] + payload = read_json(artifact) + mutation(payload) + write_json(artifact, payload) + item["sha256"] = digest(artifact) + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertTrue(any(expected in error for error in status["rejected_gates"][gate]), fault) + + def test_scanner_heal_release_bundle_requires_same_gate_measurement_window(self) -> None: + for gate, field in (("G14", "multi_pool_evidence"), ("P1", "profile_evidence"), ("P3", "heal_capacity_measurement")): + with self.subTest(gate=gate), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + evidence = data["gates"][gate]["evidence_fields"][field] + evidence["measurement_window_id"] = f"{gate.lower()}-different-window" + artifacts = [evidence] + if "profile_artifacts" in evidence: + artifacts.extend(evidence["profile_artifacts"].values()) + for item in artifacts: + artifact = bundle.parent / item["artifact"] + payload = read_json(artifact) + payload["measurement_window_id"] = evidence["measurement_window_id"] + write_json(artifact, payload) + if "measurement_window_id" in item: + item["measurement_window_id"] = evidence["measurement_window_id"] + item["sha256"] = digest(artifact) + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + expected = ( + "case_evidence[0].measurement_window_id mismatch" + if gate == "G14" + else "must share one measurement window" + ) + self.assertTrue(any(expected in error + for error in status["rejected_gates"][gate])) + + def test_scanner_heal_release_bundle_requires_domain_evidence(self) -> None: + for fault, gate, field, mutation, expected in ( + ( + "scoped-ack-cases", + "G03", + "scoped_ack_request_identity", + lambda item: item["scoped_ack_cases"].remove("exact-generation"), + "missing cases", + ), + ( + "scoped-ack-fallback", + "G03", + "scoped_ack_request_identity", + lambda item: item.update({"whole_cycle_fallback_observed": False}), + "whole_cycle_fallback_observed", + ), + ( + "scoped-ack-capability-snapshot", + "G03", + "participating_peer_capability_snapshot", + lambda item: item.pop("capability_probe_observed"), + "capability_probe_observed", + ), + ( + "scoped-ack-mixed-peer-restart", + "G03", + "mixed_peer_ack_fallback_oracle", + lambda item: item.update({"restarted_peer_rejected": False}), + "restarted_peer_rejected", + ), + ( + "durable-intent-cases", + "G04", + "root_floor_intent_crash_evidence", + lambda item: item["durable_intent_cases"].remove("persist-failure-no-202"), + "durable_intent_cases missing cases", + ), + ( + "durable-intent-persist", + "G04", + "root_floor_intent_crash_evidence", + lambda item: item.pop("persist_failure_blocks_acceptance"), + "persist_failure_blocks_acceptance", + ), + ( + "g12-reset-quota-cases", + "G12", + "reset_quota_path_evidence", + lambda item: item["quota_path_cases"].remove("future-reservation-protocol-fail-closed"), + "quota_path_cases missing cases", + ), + ( + "g12-settlement-quota-cases", + "G12", + "settlement_quota_path_evidence", + lambda item: item["quota_path_cases"].remove("rejected-object-not-visible"), + "quota_path_cases missing cases", + ), + ( + "mixed-version-cases", + "G09", + "mixed_version_writer_evidence", + lambda item: item["mixed_version_cases"].remove("old-reader-new-writer"), + "mixed_version_cases missing cases", + ), + ( + "rollback-payload", + "G09", + "rollback_payload_evidence", + lambda item: item.update({"rollback_payload_replayed": False}), + "rollback_payload_replayed", + ), + ( + "legacy-source-conflict-cases", + "R-L", + "legacy_source_conflict_evidence", + lambda item: item["legacy_source_conflict_cases"].remove("newer-target-conflict-rejected"), + "legacy_source_conflict_cases missing cases", + ), + ( + "migration-gap-closed", + "R-L", + "migration_gap_evidence", + lambda item: item.update({"migration_gap_closed": False}), + "migration_gap_closed", + ), + ( + "source-retirement-crash-safe", + "R-L", + "crash_safe_source_retirement_evidence", + lambda item: item.update({"source_retirement_is_crash_safe": False}), + "source_retirement_is_crash_safe", + ), + ( + "scheduler-bounds", + "G10", + "scheduler_bound_evidence", + lambda item: item["scheduler_bounds"].remove("lock-hold-bound"), + "scheduler_bounds missing cases", + ), + ( + "pressure-metric", + "G10", + "pressure_recovery_evidence", + lambda item: item["pressure_metrics"].pop("heal_lock_wait_p99_ms"), + "heal_lock_wait_p99_ms", + ), + ( + "cold-walk-share", + "P1", + "cold_walk_share_measurement", + lambda item: item.update({"cold_walk_share": 1.2}), + "cold_walk_share exceeds one", + ), + ( + "foreground-throughput", + "P1", + "foreground_latency_throughput_measurement", + lambda item: item.pop("throughput_ops"), + "throughput_ops", + ), + ( + "profile-cost", + "P1", + "profile_evidence", + lambda item: item.pop("save_operations"), + "save_operations", + ), + ( + "abba-leg", + "P3", + "two_hour_pressure_measurement", + lambda item: item["abba_legs"].remove("B2"), + "abba_legs missing cases", + ), + ( + "pressure-scenario", + "P3", + "two_hour_pressure_measurement", + lambda item: item["scenarios"].remove("running-heal"), + "scenarios missing cases", + ), + ( + "heal-capacity", + "P3", + "heal_capacity_measurement", + lambda item: item["heal_capacity"].pop("completed_objects"), + "completed_objects", + ), + ( + "recovery-window", + "P3", + "recovery_window_measurement", + lambda item: item["fault_modes"].remove("process-crash-restart"), + "fault_modes missing cases", + ), + ): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, bundle = self.scanner_heal_release_bundle_fixture(Path(tmp)) + data = read_json(bundle) + mutation(data["gates"][gate]["evidence_fields"][field]) + write_json(bundle, data) + + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = scanner_heal_release_bundle_status(root, bundle) + self.assertEqual(status["decision"], "blocked") + self.assertFalse(status["release_approved"]) + self.assertTrue(any(expected in error for error in status["rejected_gates"][gate]), fault) + def test_scanner_heal_case_does_not_approve_pending_release(self) -> None: with tempfile.TemporaryDirectory() as tmp: root, run_dir = self.scanner_heal_fixture(Path(tmp)) @@ -1345,14 +5090,21 @@ class SelfTests(unittest.TestCase): status = scanner_heal_release_status(root, run_dir) self.assertEqual(status["decision"], "blocked") self.assertFalse(status["release_approved"]) + self.assertTrue(status["release_schema_capable"]) self.assertEqual(status["rejected_cases"], []) self.assertEqual(len(status["pending_gates"]), 21) + self.assertIn("mixed-version-rollback", status["pending_lanes"]) + self.assertIn("ec8-4-multiset", status["pending_lanes"]) + self.assertIn("scheduler-pressure", status["pending_lanes"]) def test_scanner_heal_case_only_schema_cannot_approve_release(self) -> None: with tempfile.TemporaryDirectory() as tmp: root, run_dir = self.scanner_heal_fixture(Path(tmp)) registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["schema"] = 1 registry["release_pending"] = {} + registry.pop("release_lanes") + registry.pop("release_requirements") write_json(root / ".config/scanner-heal-required-tests.json", registry) status = scanner_heal_release_status(root, run_dir) @@ -1362,11 +5114,128 @@ class SelfTests(unittest.TestCase): self.assertEqual(status["rejected_cases"], []) self.assertEqual(status["pending_gates"], []) + def test_scanner_heal_release_requirements_cannot_be_cleared(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["release_requirements"] = [] + write_json(root / ".config/scanner-heal-required-tests.json", registry) + + with self.assertRaisesRegex(ValueError, "invalid scanner/heal release requirements"): + scanner_heal_release_status(root, run_dir) + + def test_scanner_heal_release_matrix_lanes_remain_pending_without_real_evidence(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + + errors = check_scanner_heal_evidence(root, run_dir, "release") + for gate, text in ( + ("G03", "Exact scoped ACK"), + ("G09", "mixed-version reader/writer"), + ("G14", "3x4 EC8+4"), + ("P3", "two-hour pressure/heal capacity"), + ("R-L", "Legacy source conflicts"), + ): + self.assertTrue(any(error.startswith(f"pending {gate}:") and text in error for error in errors), gate) + + status = scanner_heal_release_status(root, run_dir) + self.assertFalse(status["release_approved"]) + self.assertIn("mixed-version-rollback", status["pending_lanes"]) + self.assertIn("ec8-4-multiset", status["pending_lanes"]) + self.assertIn("scheduler-pressure", status["pending_lanes"]) + requirements, _, _ = scanner_heal_release_requirements(read_json(root / ".config/scanner-heal-required-tests.json")) + for gate, expected_fields in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS.items(): + self.assertEqual(tuple(requirements[gate]["evidence_fields"]), expected_fields, gate) + + def test_scanner_heal_required_evidence_fields_cannot_be_removed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + for gate, expected_fields in SCANNER_HEAL_RELEASE_BUNDLE_REQUIRED_EVIDENCE_FIELDS.items(): + for requirement in registry["release_requirements"]: + if requirement["gate"] == gate: + requirement["evidence_fields"] = [] + break + write_json(root / ".config/scanner-heal-required-tests.json", registry) + + with self.subTest(gate=gate): + with self.assertRaisesRegex(ValueError, f"release gate {gate} missing required evidence fields"): + scanner_heal_release_status(root, run_dir) + + registry = read_json(root / ".config/scanner-heal-required-tests.json") + for requirement in registry["release_requirements"]: + if requirement["gate"] == gate: + requirement["evidence_fields"] = list(expected_fields) + break + + def test_scanner_heal_ec84_case_rejects_wrong_erasure_geometry(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4"), []) + + path = run_dir / "background-target-crash-ec8-4.json" + oracle = read_json(path) + oracle["objects"][0]["physical"]["data_blocks"] = 10 + oracle["objects"][0]["physical"]["parity_blocks"] = 2 + oracle["objects"][0]["expected_physical"]["data_blocks"] = 10 + oracle["objects"][0]["expected_physical"]["parity_blocks"] = 2 + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0, root) + + errors = check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4") + self.assertTrue( + any("EC data/parity geometry differs from the required case" in error for error in errors), + errors, + ) + + def test_scanner_heal_multipool_case_accepts_deferred_optional_outage_write(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = run_dir / "background-target-crash-ec8-4-multi-pool.json" + oracle = read_json(path) + oracle["outage_write_deferred_until_rejoin"] = True + oracle["outage_write"] = { + "attempted": True, + "required": False, + "accepted": True, + "attempts": 36, + "service_unavailable": 36, + } + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0, root) + + self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4-multi-pool"), []) + + oracle["outage_write"]["accepted"] = False + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0, root) + errors = check_scanner_heal_evidence(root, run_dir, "background-target-crash-ec8-4-multi-pool") + self.assertTrue( + any("optional outage-write final S3 body was not accepted" in error for error in errors), + errors, + ) + + def test_scanner_heal_pending_gate_cannot_map_to_implemented_lane(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["release_requirements"][0]["lane"] = "single-set-restart" + write_json(root / ".config/scanner-heal-required-tests.json", registry) + + with self.assertRaisesRegex(ValueError, "mapped to non-pending lane"): + scanner_heal_release_status(root, run_dir) + def test_scanner_heal_release_status_rejects_synthetic_case(self) -> None: with tempfile.TemporaryDirectory() as tmp: root, run_dir = self.scanner_heal_fixture(Path(tmp)) registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["schema"] = 1 registry["release_pending"] = {} + registry.pop("release_lanes") + registry.pop("release_requirements") write_json(root / ".config/scanner-heal-required-tests.json", registry) path = run_dir / "background-target-crash.json" oracle = read_json(path) @@ -1378,6 +5247,7 @@ class SelfTests(unittest.TestCase): status = scanner_heal_release_status(root, run_dir) self.assertEqual(status["decision"], "blocked") self.assertFalse(status["release_approved"]) + self.assertFalse(status["release_schema_capable"]) self.assertEqual(status["rejected_cases"], ["background-target-crash"]) self.assertEqual(status["pending_gates"], []) @@ -1385,9 +5255,14 @@ class SelfTests(unittest.TestCase): with tempfile.TemporaryDirectory() as tmp: root, run_dir = self.scanner_heal_fixture(Path(tmp)) registry = read_json(root / ".config/scanner-heal-required-tests.json") + registry["schema"] = 1 registry["release_pending"] = {} + registry.pop("release_lanes") + registry.pop("release_requirements") write_json(root / ".config/scanner-heal-required-tests.json", registry) - (run_dir / "background-target-crash.json").unlink() + for case_id, requirement in registry["cases"].items(): + if case_id != "background-target-restart": + (run_dir / requirement["oracle"]).unlink() (run_dir / "execution.json").unlink() finish_scanner_heal_receipt(run_dir, 0, root) @@ -1395,7 +5270,7 @@ class SelfTests(unittest.TestCase): self.assertEqual(status["decision"], "blocked") self.assertFalse(status["release_approved"]) self.assertEqual(status["verified_cases"], ["background-target-restart"]) - self.assertEqual(status["rejected_cases"], ["background-target-crash"]) + self.assertEqual(status["rejected_cases"], sorted(set(registry["cases"]) - {"background-target-restart"})) def test_scanner_heal_finish_collects_oracles_from_registry(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -2245,7 +6120,10 @@ def main() -> int: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"], - ["--check-scanner-heal-release"]): + ["--check-scanner-heal-release"], ["--check-scanner-heal-release-bundle"], + ["--check-scanner-heal-release-bundle-gate"], + ["--assemble-scanner-heal-release-bundle"], + ["--write-scanner-heal-release-bundle-fixture"]): try: if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal": begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4])) @@ -2269,7 +6147,43 @@ def main() -> int: return 2 print(json.dumps(status, sort_keys=True, separators=(",", ":"))) return 0 if status["release_approved"] else 1 - raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, or --check-scanner-heal-release DIR") + if len(sys.argv) == 3 and sys.argv[1] == "--check-scanner-heal-release-bundle": + try: + status = scanner_heal_release_bundle_status(ROOT, Path(sys.argv[2])) + except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: + print(json.dumps({"schema": 1, "decision": "invalid", "release_approved": False, + "error": str(error)}, sort_keys=True, separators=(",", ":"))) + return 2 + print(json.dumps(status, sort_keys=True, separators=(",", ":"))) + return 0 if status["release_approved"] else 1 + if len(sys.argv) == 4 and sys.argv[1] == "--check-scanner-heal-release-bundle-gate": + try: + status = scanner_heal_release_bundle_gate_status(ROOT, Path(sys.argv[2]), sys.argv[3]) + except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: + print(json.dumps({"schema": 1, "decision": "invalid", "release_approved": False, + "error": str(error)}, sort_keys=True, separators=(",", ":"))) + return 2 + print(json.dumps(status, sort_keys=True, separators=(",", ":"))) + return 0 if status["decision"] == "verified" else 1 + if len(sys.argv) >= 4 and sys.argv[1] == "--assemble-scanner-heal-release-bundle": + try: + bundle, status = assemble_scanner_heal_release_bundle_descriptors( + ROOT, + [Path(item) for item in sys.argv[2:-1]], + Path(sys.argv[-1]), + ) + except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: + print(json.dumps({"schema": 1, "decision": "invalid", "release_approved": False, + "error": str(error)}, sort_keys=True, separators=(",", ":"))) + return 2 + print(bundle) + print(json.dumps(status, sort_keys=True, separators=(",", ":"))) + return 0 if status["release_approved"] else 1 + if len(sys.argv) == 3 and sys.argv[1] == "--write-scanner-heal-release-bundle-fixture": + bundle = write_scanner_heal_release_bundle_fixture(ROOT, Path(sys.argv[2])) + print(bundle) + return 0 + raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, --check-scanner-heal DIR CASE|release, --check-scanner-heal-release DIR, --check-scanner-heal-release-bundle FILE, --check-scanner-heal-release-bundle-gate FILE GATE, --assemble-scanner-heal-release-bundle DESCRIPTOR [DESCRIPTOR ...] DIR, or --write-scanner-heal-release-bundle-fixture DIR") except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error: print(f"ERROR: {error}", file=sys.stderr) return 1 @@ -2296,7 +6210,10 @@ def main() -> int: if sys.argv[1:]: print( "usage: check_test_wiring.py [--self-test | --check-core LISTING | --check-profile PROFILE LISTING | " - "--update-profile PROFILE LISTING PLATFORM]", + "--update-profile PROFILE LISTING PLATFORM | --check-scanner-heal-release-bundle FILE | " + "--check-scanner-heal-release-bundle-gate FILE GATE | " + "--assemble-scanner-heal-release-bundle DESCRIPTOR [DESCRIPTOR ...] DIR | " + "--write-scanner-heal-release-bundle-fixture DIR]", file=sys.stderr, ) return 2 diff --git a/scripts/ci_gate.py b/scripts/ci_gate.py index 4c0ae574c..74442691f 100644 --- a/scripts/ci_gate.py +++ b/scripts/ci_gate.py @@ -59,7 +59,7 @@ def expected_results(mode: str, event: str, ref: str) -> dict[str, str]: expected.update({job: "success" if mode == "full" else "skipped" for job in CODE_JOBS}) rio = mode == "full" and event in ("schedule", "workflow_dispatch") expected.update({job: "success" if rio else "skipped" for job in OPTIONAL_JOBS[:2]}) - full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref == "refs/heads/main")) + full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref in ("refs/heads/main", "refs/heads/release"))) expected["e2e-full"] = "success" if full else "skipped" return expected @@ -219,6 +219,27 @@ class SelfTests(unittest.TestCase): bad = {**good, "classify-changes": {"result": "success", "outputs": selection}} self.assertTrue(verify_results(bad, event, "refs/heads/main")) + def test_full_e2e_gate_preserves_workflow_branch_and_event_scope(self): + for event, ref, required in ( + ("push", "refs/heads/main", "success"), + ("push", "refs/heads/release", "success"), + ("push", "refs/heads/feature", "skipped"), + ("push", "refs/heads/release-candidate", "skipped"), + ("push", "refs/tags/release", "skipped"), + ("pull_request", "refs/pull/1/merge", "skipped"), + ("schedule", "refs/heads/release", "skipped"), + ("workflow_dispatch", "refs/heads/feature", "success"), + ("merge_group", "refs/heads/gh-readonly-queue/release/pr-1", "success"), + ): + with self.subTest(event=event, ref=ref): + expected = expected_results("full", event, ref) + self.assertEqual(expected["e2e-full"], required) + needs = {job: {"result": result} for job, result in expected.items()} + needs["classify-changes"]["outputs"] = {"mode": "full"} + for result in ("success", "skipped", "failure", "cancelled"): + needs["e2e-full"]["result"] = result + self.assertEqual(verify_results(needs, event, ref) == [], result == required) + def test_repository_wiring_and_missing_dependency_regression(self): self.assertEqual(check_workflow(ROOT), []) with tempfile.TemporaryDirectory() as directory: diff --git a/scripts/diagnose_scanner_enumeration_restart.py b/scripts/diagnose_scanner_enumeration_restart.py index 3be70cdfb..95d8b2a6e 100644 --- a/scripts/diagnose_scanner_enumeration_restart.py +++ b/scripts/diagnose_scanner_enumeration_restart.py @@ -74,10 +74,31 @@ def converged(report, objects): def replays_raw_window(previous, current): + if (previous["raw_entries"] == 0 or current["raw_entries"] == 0 + or previous["raw_first_entry"] is None or current["raw_first_entry"] is None + or previous["raw_last_entry"] is None or current["raw_last_entry"] is None): + return False + raw_page_index_progressed = ( + previous["raw_page_index_parent"] == current["raw_page_index_parent"] + and ( + current["raw_page_index_committed_entries"] > previous["raw_page_index_committed_entries"] + or (current["raw_page_index_complete"] and not previous["raw_page_index_complete"]) + ) + ) return (previous["raw_first_entry"] == current["raw_first_entry"] and previous["raw_last_entry"] == current["raw_last_entry"] and previous["objects_retained"] == current["objects_before"] - and current["objects_retained"] == previous["objects_retained"]) + and current["objects_retained"] == previous["objects_retained"] + and not raw_page_index_progressed) + + +def fully_retained(report, objects): + return all(report[key] == objects for key in + ("objects_retained", "versions_retained", "bytes_retained")) + + +def final_complete_recheck_after_full_retention(previous, current, objects): + return fully_retained(previous, objects) and converged(current, objects) def validate_recoverable_quantum(reports, *, objects, budget, require_converged): @@ -97,7 +118,8 @@ def validate_recoverable_quantum(reports, *, objects, budget, require_converged) raise ValueError("durable retained coverage did not survive process restart") if report["objects_retained"] < previous["objects_retained"]: raise ValueError("durable retained coverage regressed across restart") - if replays_raw_window(previous, report): + if (not final_complete_recheck_after_full_retention(previous, report, objects) + and replays_raw_window(previous, report)): raise ValueError("raw enumeration window replayed without durable coverage") if (report["raw_page_index_parent"] == previous["raw_page_index_parent"] and report["raw_page_index_committed_entries"] < previous["raw_page_index_committed_entries"] diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index b21e757c4..7b7abe72a 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -26,7 +26,7 @@ 6|crates/ecstore/src/config/com.rs 14|crates/ecstore/src/config/storageclass.rs 178|crates/ecstore/src/core/pools.rs -7|crates/ecstore/src/data_movement/mod.rs +6|crates/ecstore/src/data_movement/mod.rs 2|crates/ecstore/src/data_usage/local_snapshot.rs 12|crates/ecstore/src/data_usage/mod.rs 5|crates/ecstore/src/disk/local.rs diff --git a/scripts/python_bin.sh b/scripts/python_bin.sh index b36b35a05..ed89b6d4a 100755 --- a/scripts/python_bin.sh +++ b/scripts/python_bin.sh @@ -29,7 +29,7 @@ MIN_MINOR=11 UV_PYTHON_SPEC="${RUSTFS_UV_PYTHON:-3.12}" version_ok() { - "$1" -c "import sys; raise SystemExit(0 if sys.version_info >= (${MIN_MAJOR}, ${MIN_MINOR}) else 1)" \ + "$@" -c "import sys; raise SystemExit(0 if sys.version_info >= (${MIN_MAJOR}, ${MIN_MINOR}) else 1)" \ >/dev/null 2>&1 } @@ -55,20 +55,25 @@ if [ "${1:-}" = "--print-interpreter" ]; then fi if [ -n "${RUSTFS_PYTHON:-}" ]; then - if ! command -v "${RUSTFS_PYTHON}" >/dev/null 2>&1; then + read -r -a rustfs_python_cmd <<< "${RUSTFS_PYTHON}" + if [ "${#rustfs_python_cmd[@]}" -eq 0 ]; then echo >&2 "❌ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is not an executable command." exit 1 fi - if ! version_ok "${RUSTFS_PYTHON}"; then + if ! command -v "${rustfs_python_cmd[0]}" >/dev/null 2>&1; then + echo >&2 "❌ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is not an executable command." + exit 1 + fi + if ! version_ok "${rustfs_python_cmd[@]}"; then echo >&2 "❌ RUSTFS_PYTHON='${RUSTFS_PYTHON}' is older than Python ${MIN_MAJOR}.${MIN_MINOR}." echo >&2 " The repository's checkers import tomllib (Python ${MIN_MAJOR}.${MIN_MINOR}+)." exit 1 fi if [ "${print_only}" = "1" ]; then - command -v "${RUSTFS_PYTHON}" + printf '%s\n' "${RUSTFS_PYTHON}" exit 0 fi - exec "${RUSTFS_PYTHON}" "$@" + exec "${rustfs_python_cmd[@]}" "$@" fi for candidate in python3.14 python3.13 python3.12 python3.11 python3 python; do diff --git a/scripts/run_scanner_heal_authority_evidence.py b/scripts/run_scanner_heal_authority_evidence.py new file mode 100755 index 000000000..78f15cd2b --- /dev/null +++ b/scripts/run_scanner_heal_authority_evidence.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal G01 authority release evidence. + +The producer consumes operator-collected measured JSON. It packages root and +quota authority measurements into the common release-bundle descriptor shape +and lets check_test_wiring.py validate G01 without approving the full release. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import check_test_wiring as wiring + +ROOT = Path(__file__).resolve().parents[1] + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def timestamp(value: Any, name: str) -> str: + wiring.require(isinstance(value, str) and value.strip(), f"missing {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + wiring.require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def measured_json(path: Path, source_revision: str, label: str) -> dict[str, Any]: + payload = wiring.read_json(path.resolve()) + wiring.require(isinstance(payload, dict), f"{label} must be a JSON object") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(payload.get(marker) is not True, f"{label} is {marker}") + wiring.require(payload.get("schema") == 1, f"{label} schema must be 1") + wiring.require(payload.get("evidence_type") == "measured", f"{label} must be measured") + wiring.require(payload.get("source_revision") == source_revision, f"{label} source revision mismatch") + return payload + + +def common_evidence(args: argparse.Namespace, source_revision: str) -> dict[str, Any]: + duration = wiring.evidence_integer(args.duration_seconds, "duration_seconds", 1, 86400) + started_at = timestamp(args.started_at, "started_at") if args.started_at else datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + if args.finished_at: + finished_at = timestamp(args.finished_at, "finished_at") + else: + started = datetime.fromisoformat(started_at.replace("Z", "+00:00")) + finished_at = (started + timedelta(seconds=duration)).isoformat().replace("+00:00", "Z") + wiring.require( + datetime.fromisoformat(finished_at.replace("Z", "+00:00")) >= datetime.fromisoformat(started_at.replace("Z", "+00:00")), + "authority measurement timestamps are inverted", + ) + run_id = args.run_id or f"authority-{source_revision[:12]}" + window_id = args.measurement_window_id or f"authority-window-{source_revision[:12]}" + wiring.evidence_string(run_id, "run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + wiring.evidence_string(window_id, "measurement_window_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + wiring.require(run_id != window_id, "authority proof must separate run/window identities") + return { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "started_at": started_at, + "finished_at": finished_at, + "command": [ + "scripts/run_scanner_heal_authority_evidence.py", + "--root-authority-json", + "", + "--quota-authority-json", + "", + ], + } + + +def validate_root_authority(payload: dict[str, Any]) -> None: + wiring.release_bundle_exact_strings( + payload.get("root_authority_cases"), + wiring.SCANNER_HEAL_RELEASE_G01_ROOT_AUTHORITY_CASES, + "root_authority_cases", + ) + wiring.release_bundle_bool_true(payload.get("root_cas_observed"), "root_cas_observed") + wiring.release_bundle_bool_true(payload.get("root_readback_observed"), "root_readback_observed") + wiring.release_bundle_bool_true(payload.get("incomplete_root_rejected"), "incomplete_root_rejected") + wiring.release_bundle_bool_true(payload.get("stale_root_rejected"), "stale_root_rejected") + + +def validate_quota_authority(payload: dict[str, Any]) -> None: + wiring.release_bundle_exact_strings( + payload.get("quota_authority_cases"), + wiring.SCANNER_HEAL_RELEASE_G01_QUOTA_AUTHORITY_CASES, + "quota_authority_cases", + ) + wiring.release_bundle_bool_true(payload.get("quota_floor_readback_observed"), "quota_floor_readback_observed") + wiring.release_bundle_bool_true(payload.get("over_limit_put_rejected"), "over_limit_put_rejected") + wiring.release_bundle_bool_true(payload.get("rejected_object_invisible"), "rejected_object_invisible") + wiring.release_bundle_bool_true(payload.get("quota_fails_closed_without_authority"), "quota_fails_closed_without_authority") + + +def write_field(out_dir: Path, field: str, common: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: + evidence = { + **payload, + **common, + "evidence_type": "measured", + "summary": payload.get("summary") or f"Measured Scanner/Heal G01.{field} evidence.", + } + artifact = out_dir / "artifacts" / f"G01-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact_payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "gate": "G01", + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + artifact_payload[key] = value + wiring.write_json(artifact, artifact_payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = wiring.digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + wiring.require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + root_authority = measured_json(args.root_authority_json, source_revision, "root authority artifact") + quota_authority = measured_json(args.quota_authority_json, source_revision, "quota authority artifact") + validate_root_authority(root_authority) + validate_quota_authority(quota_authority) + out_dir.mkdir(parents=True) + common = common_evidence(args, source_revision) + fields = { + "root_authority_evidence": write_field(out_dir, "root_authority_evidence", common, root_authority), + "quota_authority_evidence": write_field(out_dir, "quota_authority_evidence", common, quota_authority), + } + descriptor = out_dir / "release-bundle-authority.json" + wiring.write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": { + "G01": { + "status": "pass", + "lane": "authority-coverage", + "evidence_type": "measured", + "evidence_fields": fields, + }, + }, + }) + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + "G01", + ], cwd=ROOT) + return descriptor + + +def write_self_test_inputs(root: Path, source_revision: str) -> tuple[Path, Path]: + root_json = root / "root-authority.json" + quota_json = root / "quota-authority.json" + wiring.write_json(root_json, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "root_authority_cases": list(wiring.SCANNER_HEAL_RELEASE_G01_ROOT_AUTHORITY_CASES), + "root_cas_observed": True, + "root_readback_observed": True, + "incomplete_root_rejected": True, + "stale_root_rejected": True, + }) + wiring.write_json(quota_json, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "quota_authority_cases": list(wiring.SCANNER_HEAL_RELEASE_G01_QUOTA_AUTHORITY_CASES), + "quota_floor_readback_observed": True, + "over_limit_put_rejected": True, + "rejected_object_invisible": True, + "quota_fails_closed_without_authority": True, + }) + return root_json, quota_json + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + root_json, quota_json = write_self_test_inputs(root, source_revision) + descriptor = build_descriptor(parse_args([ + "--root-authority-json", str(root_json), + "--quota-authority-json", str(quota_json), + "--out-dir", str(root / "out"), + "--duration-seconds", "60", + ])) + wiring.require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + root_json, quota_json = write_self_test_inputs(root, source_revision) + payload = wiring.read_json(root_json) + payload["synthetic"] = True + wiring.write_json(root_json, payload) + try: + build_descriptor(parse_args([ + "--root-authority-json", str(root_json), + "--quota-authority-json", str(quota_json), + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + wiring.require("synthetic" in str(err), "wrong self-test failure for synthetic root authority") + else: + raise ValueError("self-test accepted synthetic root authority") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + root_json, quota_json = write_self_test_inputs(root, source_revision) + payload = wiring.read_json(quota_json) + payload["quota_authority_cases"] = payload["quota_authority_cases"][:-1] + wiring.write_json(quota_json, payload) + try: + build_descriptor(parse_args([ + "--root-authority-json", str(root_json), + "--quota-authority-json", str(quota_json), + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + wiring.require("missing cases" in str(err), "wrong self-test failure for incomplete quota authority") + else: + raise ValueError("self-test accepted incomplete quota authority") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root-authority-json", type=Path) + parser.add_argument("--quota-authority-json", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--run-id") + parser.add_argument("--measurement-window-id") + parser.add_argument("--started-at") + parser.add_argument("--finished-at") + parser.add_argument("--duration-seconds", type=int, default=60) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.root_authority_json is None: + parser.error("--root-authority-json is required unless --self-test is used") + if args.quota_authority_json is None: + parser.error("--quota-authority-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Authority release descriptor verified: {descriptor}") + return 0 + except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_checkpoint_crash_evidence.py b/scripts/run_scanner_heal_checkpoint_crash_evidence.py new file mode 100755 index 000000000..6e14fd930 --- /dev/null +++ b/scripts/run_scanner_heal_checkpoint_crash_evidence.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +"""Assemble Scanner/Heal G02/R-E release evidence from measured restart diagnostics.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +from scanner_abba import digest, read_json, require, write_json + +ROOT = Path(__file__).resolve().parents[1] +CHECKPOINT_FIELDS = ("bounded_checkpoint_oracle", "independent_version_inventory") +RESTART_FIELDS = ("fixed_budget_restart_evidence", "enumeration_evidence", "classification_evidence") + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def positive_int(value: Any, name: str, minimum: int = 1) -> int: + require(type(value) is int and value >= minimum, f"invalid {name}") + return value + + +def reject_non_measured_markers(payload: dict[str, Any], label: str) -> None: + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(payload.get(marker) is not True, f"{label} is {marker}") + + +def timestamp(value: Any, name: str) -> str: + require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}") + datetime.fromisoformat(value.replace("Z", "+00:00")) + return value + + +def round_report_paths(directory: Path) -> list[Path]: + paths: list[tuple[int, Path]] = [] + for path in directory.glob("round-*.json"): + match = re.fullmatch(r"round-(\d+)\.json", path.name) + require(match is not None, f"invalid round report name: {path.name}") + paths.append((int(match.group(1)), path)) + return [path for _, path in sorted(paths)] + + +def load_reports(directory: Path) -> list[dict[str, Any]]: + require(directory.is_dir(), "diagnostic directory is missing") + reports = [read_json(path) for path in round_report_paths(directory)] + require(reports, "diagnostic directory has no round reports") + non_negative_counters = { + "raw_entries", + "raw_name_bytes", + "objects_before", + "objects_retained", + "versions_retained", + "bytes_retained", + "objects_processed", + "raw_page_index_committed_entries", + "raw_page_index_indexed_entries", + } + for index, report in enumerate(reports): + reject_non_measured_markers(report, f"round {index}") + require(report.get("schema") == 1, f"round {index} has wrong schema") + require(report.get("round") == index, f"round {index} order mismatch") + for key in ( + "pid", + "objects_expected", + "raw_entry_budget", + "raw_entries", + "raw_name_bytes", + "objects_before", + "objects_retained", + "versions_retained", + "bytes_retained", + "objects_processed", + "raw_page_index_committed_entries", + "raw_page_index_indexed_entries", + ): + positive_int(report.get(key), f"round {index} {key}", 0 if key in non_negative_counters else 1) + require(type(report.get("snapshot_complete")) is bool, f"round {index} missing snapshot_complete") + require(type(report.get("raw_page_index_complete")) is bool, f"round {index} missing raw_page_index_complete") + require(report.get("outcome") in {"complete", "partial", "cancelled_without_cache"}, f"round {index} bad outcome") + return reports + + +def require_measured_manifest(path: Path, source_revision: str) -> dict[str, Any]: + manifest = read_json(path) + reject_non_measured_markers(manifest, "checkpoint/crash manifest") + require(manifest.get("schema") == 1, "unsupported manifest schema") + require(manifest.get("evidence_type") == "measured", "manifest must be measured") + require(manifest.get("source_revision") == source_revision, "manifest source revision mismatch") + require(isinstance(manifest.get("run_id"), str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", manifest["run_id"]), + "invalid run_id") + require(isinstance(manifest.get("measurement_window_id"), str) + and manifest["measurement_window_id"] != manifest["run_id"], "invalid measurement_window_id") + timestamp(manifest.get("started_at"), "started_at") + timestamp(manifest.get("finished_at"), "finished_at") + require(isinstance(manifest.get("command"), list) and manifest["command"], "missing command provenance") + require(manifest.get("diagnostic_exit_code") == 0, "diagnostic did not pass") + return manifest + + +def derived_measured_manifest(directory: Path, source_revision: str, reports: list[dict[str, Any]]) -> dict[str, Any]: + request_path = directory / "request.json" + request = read_json(request_path) + require(isinstance(request, dict), "diagnostic request must be a JSON object") + reject_non_measured_markers(request, "diagnostic request") + objects = positive_int(request.get("objects"), "request.objects") + raw_entry_budget = positive_int(request.get("raw_entry_budget"), "request.raw_entry_budget") + final_round = positive_int(request.get("round"), "request.round", 0) + require(objects == reports[-1]["objects_expected"], "diagnostic request object count mismatch") + require(raw_entry_budget == reports[-1]["raw_entry_budget"], "diagnostic request raw budget mismatch") + require(final_round == reports[-1]["round"], "diagnostic request final round mismatch") + paths = [request_path] + round_report_paths(directory) + started = datetime.fromtimestamp(min(path.stat().st_mtime for path in paths), timezone.utc).replace(microsecond=0) + finished = datetime.fromtimestamp(max(path.stat().st_mtime for path in paths), timezone.utc).replace(microsecond=0) + run_id = re.sub(r"[^A-Za-z0-9._:-]", "-", directory.name) + require(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", run_id) is not None, + "diagnostic directory name cannot be used as run_id") + return { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": f"{run_id}-window", + "started_at": started.isoformat().replace("+00:00", "Z"), + "finished_at": finished.isoformat().replace("+00:00", "Z"), + "command": [ + "python3", + "scripts/diagnose_scanner_enumeration_restart.py", + "--test-binary", + "", + "--output", + str(directory), + "--objects", + str(objects), + "--raw-entry-budget", + str(raw_entry_budget), + "--rounds", + str(final_round + 1), + ], + "diagnostic_exit_code": 0, + } + + +def summarize(reports: list[dict[str, Any]]) -> dict[str, Any]: + first = reports[0] + final = reports[-1] + objects_expected = positive_int(final["objects_expected"], "objects_expected") + raw_entry_budget = positive_int(final["raw_entry_budget"], "raw_entry_budget") + require(final.get("snapshot_complete") is True and final.get("outcome") == "complete", + "fixed-budget restart convergence was not established") + require(final.get("objects_retained") == objects_expected, "final retained objects mismatch") + require(final.get("versions_retained") == objects_expected, "final retained versions mismatch") + require(final.get("bytes_retained") == objects_expected, "final retained bytes mismatch") + pids = {positive_int(report["pid"], "pid") for report in reports} + require(len(pids) >= 2 or len(reports) >= 2, "restart diagnostic must include at least two worker rounds") + require(any(report.get("raw_entries", 0) > 0 for report in reports), "raw enumeration was not observed") + require(any(report.get("raw_page_index_committed_entries", 0) > 0 for report in reports), + "durable raw page commit was not observed") + require(any(report.get("objects_processed", 0) > 0 for report in reports), "classification was not observed") + previous = None + frontier_retained = False + for report in reports: + require(report["raw_entries"] <= raw_entry_budget, "raw-entry budget exceeded") + require(report["objects_processed"] <= raw_entry_budget, "object budget exceeded") + if previous is not None: + require(report["objects_before"] == previous["objects_retained"], + "retained coverage did not survive restart") + frontier_retained |= report["objects_before"] >= previous["objects_retained"] + previous = report + return { + "objects_expected": objects_expected, + "raw_entry_budget": raw_entry_budget, + "max_raw_entries_per_round": max(report["raw_entries"] for report in reports), + "max_objects_processed_per_round": max(report["objects_processed"] for report in reports), + "object_processing_attempts": sum(report["objects_processed"] for report in reports), + "objects_processed": final["objects_retained"], + "objects_retained": final["objects_retained"], + "versions_retained": final["versions_retained"], + "bytes_retained": final["bytes_retained"], + "restart_rounds": len(reports), + "raw_page_index_complete": max(report["raw_page_index_committed_entries"] for report in reports) >= objects_expected + and max(report["raw_page_index_indexed_entries"] for report in reports) >= objects_expected, + "enumeration_frontier_retained": frontier_retained, + "first_round": first, + "final_round": final, + } + + +def write_field(out_dir: Path, gate: str, field: str, evidence: dict[str, Any]) -> dict[str, Any]: + artifact = out_dir / "artifacts" / f"{gate}-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "artifact_kind": "scanner-checkpoint-crash-evidence", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "started_at": evidence["started_at"], + "finished_at": evidence["finished_at"], + "gate": gate, + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "command"}: + payload[key] = value + write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + reports = load_reports(args.diagnostic_dir.resolve()) + if args.manifest is None: + manifest = derived_measured_manifest(args.diagnostic_dir.resolve(), source_revision, reports) + else: + manifest = require_measured_manifest(args.manifest.resolve(), source_revision) + summary = summarize(reports) + out_dir.mkdir(parents=True) + common = { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": manifest["run_id"], + "measurement_window_id": manifest["measurement_window_id"], + "started_at": manifest["started_at"], + "finished_at": manifest["finished_at"], + "command": manifest["command"], + } + gates = { + "G02": { + "status": "pass", + "lane": "checkpoint-and-crash", + "evidence_type": "measured", + "evidence_fields": { + "bounded_checkpoint_oracle": write_field(out_dir, "G02", "bounded_checkpoint_oracle", { + **common, + "summary": "Measured scanner-worker restart reports bounded raw enumeration and object processing.", + "checkpoint_progress_bounded": True, + "raw_entry_budget": summary["raw_entry_budget"], + "max_raw_entries_per_round": summary["max_raw_entries_per_round"], + "max_objects_processed_per_round": summary["max_objects_processed_per_round"], + "bounded_work_quantum_observed": True, + "durable_checkpoint_committed": True, + "no_unbounded_tail": True, + }), + "independent_version_inventory": write_field(out_dir, "G02", "independent_version_inventory", { + **common, + "summary": "Measured restart convergence retained an independent object/version/byte inventory.", + "independent_version_inventory_observed": True, + "objects_expected": summary["objects_expected"], + "objects_retained": summary["objects_retained"], + "versions_retained": summary["versions_retained"], + "bytes_retained": summary["bytes_retained"], + }), + }, + }, + "R-E": { + "status": "pass", + "lane": "checkpoint-and-crash", + "evidence_type": "measured", + "evidence_fields": { + "fixed_budget_restart_evidence": write_field(out_dir, "R-E", "fixed_budget_restart_evidence", { + **common, + "summary": "Measured scanner worker converged after repeated process restarts without an unbudgeted final sweep.", + "crash_points": ["scanner-worker-process-restart"], + "fixed_budget_restart_converged": True, + "restart_rounds": summary["restart_rounds"], + "raw_entry_budget": summary["raw_entry_budget"], + "no_unbudgeted_final_sweep": True, + }), + "enumeration_evidence": write_field(out_dir, "R-E", "enumeration_evidence", { + **common, + "summary": "Measured raw enumeration and raw-page checkpoint progress survived worker restarts.", + "crash_points": ["scanner-worker-process-restart"], + "raw_enumeration_observed": True, + "durable_raw_page_commit_observed": True, + "raw_page_index_complete": summary["raw_page_index_complete"], + "enumeration_frontier_retained": summary["enumeration_frontier_retained"], + }), + "classification_evidence": write_field(out_dir, "R-E", "classification_evidence", { + **common, + "summary": "Measured object classification and retained inventory converged under the fixed restart budget.", + "crash_points": ["scanner-worker-process-restart"], + "classification_observed": True, + "objects_processed": summary["objects_processed"], + "object_processing_attempts": summary["object_processing_attempts"], + "objects_retained": summary["objects_retained"], + "versions_retained": summary["versions_retained"], + "bytes_retained": summary["bytes_retained"], + }), + }, + }, + } + descriptor = out_dir / "release-bundle-checkpoint-crash.json" + write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + for gate in ("G02", "R-E"): + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + gate, + ], cwd=ROOT) + return descriptor + + +def write_self_test_inputs( + root: Path, + source_revision: str, + complete: bool = True, + objects_expected: int = 16, +) -> tuple[Path, Path]: + diagnostic = root / "diagnostic" + diagnostic.mkdir() + raw_entry_budget = 8 + reports = [] + objects_retained = 0 + for round_index in range((objects_expected + raw_entry_budget - 1) // raw_entry_budget): + remaining = objects_expected - objects_retained + processed = min(raw_entry_budget, remaining) + is_final = objects_retained + processed >= objects_expected + retained_after = objects_retained + processed + if is_final and not complete: + retained_after = max(objects_retained, objects_expected - raw_entry_budget // 2) + reports.append({ + "schema": 1, + "round": round_index, + "pid": 1000 + round_index, + "objects_expected": objects_expected, + "raw_entry_budget": raw_entry_budget, + "raw_entries": processed, + "raw_name_bytes": 128, + "objects_before": objects_retained, + "objects_retained": retained_after, + "versions_retained": retained_after, + "bytes_retained": retained_after, + "objects_processed": processed, + "raw_page_index_parent": "bucket", + "raw_page_index_committed_entries": min(objects_expected, retained_after), + "raw_page_index_indexed_entries": min(objects_expected, retained_after), + "raw_page_index_complete": is_final and complete, + "snapshot_complete": is_final and complete, + "outcome": "complete" if is_final and complete else "partial", + }) + objects_retained = retained_after + for report in reports: + write_json(diagnostic / f"round-{report['round']}.json", report) + write_json(diagnostic / "request.json", { + "workspace": str(diagnostic), + "objects": reports[-1]["objects_expected"], + "raw_entry_budget": reports[-1]["raw_entry_budget"], + "round": reports[-1]["round"], + }) + started = datetime.now(timezone.utc).replace(microsecond=0) + manifest = root / "manifest.json" + write_json(manifest, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": "checkpoint-crash-self-test-run", + "measurement_window_id": "checkpoint-crash-self-test-window", + "started_at": started.isoformat().replace("+00:00", "Z"), + "finished_at": (started + timedelta(minutes=5)).isoformat().replace("+00:00", "Z"), + "command": ["scripts/diagnose_scanner_enumeration_restart.py", "--test-binary", ""], + "diagnostic_exit_code": 0, + }) + return manifest, diagnostic + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + manifest, diagnostic = write_self_test_inputs(root, source_revision) + descriptor = build_descriptor(parse_args([ + "--manifest", str(manifest), + "--diagnostic-dir", str(diagnostic), + "--out-dir", str(root / "out"), + ])) + require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + _, diagnostic = write_self_test_inputs(root, source_revision, objects_expected=96) + descriptor = build_descriptor(parse_args([ + "--diagnostic-dir", str(diagnostic), + "--out-dir", str(root / "out"), + ])) + require(descriptor.is_file(), "self-test descriptor missing for derived manifest") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + manifest, diagnostic = write_self_test_inputs(root, source_revision, complete=False) + try: + build_descriptor(parse_args([ + "--manifest", str(manifest), + "--diagnostic-dir", str(diagnostic), + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("convergence" in str(err), "wrong self-test failure for non-converged diagnostic") + else: + raise ValueError("self-test accepted non-converged diagnostic") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + manifest, diagnostic = write_self_test_inputs(root, source_revision) + report = read_json(diagnostic / "round-0.json") + report["synthetic"] = True + write_json(diagnostic / "round-0.json", report) + try: + build_descriptor(parse_args([ + "--manifest", str(manifest), + "--diagnostic-dir", str(diagnostic), + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("round 0 is synthetic" in str(err), "wrong self-test failure for synthetic round report") + else: + raise ValueError("self-test accepted synthetic diagnostic round report") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path) + parser.add_argument("--diagnostic-dir", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.diagnostic_dir is None: + parser.error("--diagnostic-dir is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Checkpoint/crash release descriptor verified: {descriptor}") + return 0 + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_evidence_case.sh b/scripts/run_scanner_heal_evidence_case.sh index ff7c30086..7db8bead9 100755 --- a/scripts/run_scanner_heal_evidence_case.sh +++ b/scripts/run_scanner_heal_evidence_case.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" -PROFILE="e2e-nightly" +PROFILE="" CASE_ID="background-target-crash" RUN_DIR="" PLAN_ONLY=0 @@ -18,7 +18,7 @@ then validate the produced receipt, nextest listing, JUnit, and case oracle. Options: --case CASE Registry case to run (default: background-target-crash) - --profile PROFILE Nextest profile to use (default: e2e-nightly) + --profile PROFILE Nextest profile to use (default: registry lane) --run-dir DIR New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP) --plan-only Validate registry selection and print the exact filter without running cargo --self-test Run lightweight CLI/registry checks without building Rust @@ -28,9 +28,24 @@ The script intentionally runs a single case, not the release pseudo-case. After a successful case run it verifies that the release gate still remains blocked. Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e port allocator when the default 20000..30000 test range is unavailable. +Set RUSTFS_SCANNER_HEAL_SKIP_CLEAN=1 to reuse an existing cargo target directory +while narrowing a case locally; release evidence should keep the default clean +build. USAGE } +case_ids() { + "$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" <<'PY' +import json +import pathlib +import sys + +registry = json.loads(pathlib.Path(sys.argv[1]).read_text()) +for case_id in sorted(registry["cases"]): + print(case_id) +PY +} + case_field() { local case_id="$1" local field="$2" @@ -62,6 +77,48 @@ print("test(/^" + re.escape(case["name"]) + "$/)") PY } +runtime_profile_for() { + local case_id="$1" + case "$case_id" in + background-target-crash|background-target-restart) + echo "background-4x1" + ;; + background-target-crash-ec8-4|background-target-restart-ec8-4|background-target-restart-ec8-4-multi-set) + echo "background-ec8-4" + ;; + background-target-crash-ec8-4-multi-pool) + echo "background-ec8-4-multi-pool" + ;; + ec84-target-drive-restart) + echo "distributed-ec8-4" + ;; + *) + echo "default" + ;; + esac +} + +apply_runtime_profile() { + local case_id="$1" + case "$(runtime_profile_for "$case_id")" in + background-4x1) + export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" + export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}" + export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-120}" + ;; + background-ec8-4) + export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-32}" + export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-8388608}" + export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-180}" + ;; + background-ec8-4-multi-pool) + export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" + export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-4194304}" + export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-240}" + ;; + esac +} + test_binary_from_listing() { local listing="$1" local case_id="$2" @@ -103,26 +160,32 @@ import sys status = json.loads(pathlib.Path(sys.argv[1]).read_text()) if status.get("decision") != "blocked" or status.get("release_approved") is not False: raise SystemExit("release status did not record a blocked decision") -if status.get("release_schema_capable") is not False: - raise SystemExit("case-only evidence schema unexpectedly became release-capable") +if not status.get("pending_gates"): + raise SystemExit("release status did not retain pending gates") PY } run_self_test() { - local filter - filter="$(test_filter_for background-target-crash)" - case "$filter" in - *background_target_crash*) ;; - *) - echo "self-test failed: crash case filter missing" >&2 - return 1 - ;; - esac if "$0" --case release --plan-only >/dev/null 2>&1; then echo "self-test failed: release pseudo-case must not be runnable" >&2 return 1 fi - "$0" --case background-target-crash --plan-only >/dev/null + local case_id + while IFS= read -r case_id; do + local expected_filter expected_profile plan + expected_filter="$(test_filter_for "$case_id")" + expected_profile="$(case_field "$case_id" lane)" + expected_runtime_profile="$(runtime_profile_for "$case_id")" + plan="$("$0" --case "$case_id" --plan-only)" + if [[ "$plan" != *"case=$case_id"* ]] || + [[ "$plan" != *"profile=$expected_profile"* ]] || + [[ "$plan" != *"runtime_profile=$expected_runtime_profile"* ]] || + [[ "$plan" != *"filter=$expected_filter"* ]] || + [[ "$plan" != *"run_dir=$ROOT/target/scanner-heal-evidence/$case_id-"* ]]; then + echo "self-test failed: registry case plan mismatch for $case_id" >&2 + return 1 + fi + done < <(case_ids) } while [[ $# -gt 0 ]]; do @@ -166,13 +229,10 @@ fi case_field "$CASE_ID" name >/dev/null TEST_FILTER="$(test_filter_for "$CASE_ID")" -case "$CASE_ID" in - background-target-crash|background-target-restart) - export RUSTFS_HEAL_CHAOS_OBJECT_COUNT="${RUSTFS_HEAL_CHAOS_OBJECT_COUNT:-64}" - export RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES="${RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES:-16777216}" - export RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS="${RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS:-120}" - ;; -esac +if [[ -z "$PROFILE" ]]; then + PROFILE="$(case_field "$CASE_ID" lane)" +fi +apply_runtime_profile "$CASE_ID" if [[ -z "$RUN_DIR" ]]; then RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)" elif [[ "$RUN_DIR" != /* ]]; then @@ -182,6 +242,7 @@ fi if [[ "$PLAN_ONLY" == 1 ]]; then echo "case=$CASE_ID" echo "profile=$PROFILE" + echo "runtime_profile=$(runtime_profile_for "$CASE_ID")" echo "filter=$TEST_FILTER" echo "run_dir=$RUN_DIR" exit 0 @@ -197,18 +258,31 @@ if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then echo "commit tracked source changes before creating evidence" >&2 exit 1 fi +NOFILE_SOFT="$(ulimit -Sn)" +NOFILE_HARD="$(ulimit -Hn)" +if [[ "$NOFILE_SOFT" =~ ^[0-9]+$ && "$NOFILE_HARD" =~ ^[0-9]+$ && "$NOFILE_SOFT" -lt 65535 ]]; then + if [[ "$NOFILE_HARD" -ge 65535 ]]; then + ulimit -n 65535 || true + elif [[ "$NOFILE_HARD" -gt "$NOFILE_SOFT" ]]; then + ulimit -n "$NOFILE_HARD" || true + fi +fi mkdir -p "$(dirname "$RUN_DIR")" TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")" trap 'rm -rf "$TMP_DIR"' EXIT BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}" -cargo clean -p rustfs +TARGET_DIR="${CARGO_TARGET_DIR:-$ROOT/target}" +DEBUG_DIR="$TARGET_DIR/debug" +if [[ "${RUSTFS_SCANNER_HEAL_SKIP_CLEAN:-0}" != "1" ]]; then + cargo clean -p rustfs +fi if [[ -n "$BUILD_FEATURES" ]]; then cargo build --locked -p rustfs --bins --features "$BUILD_FEATURES" else cargo build --locked -p rustfs --bins fi -printf '%s' "$BUILD_FEATURES" >"$ROOT/target/debug/rustfs.features" +printf '%s' "$BUILD_FEATURES" >"$DEBUG_DIR/rustfs.features" LISTING_TMP="$TMP_DIR/listing.json" NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ @@ -220,12 +294,13 @@ cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message- TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")" export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}" -"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$ROOT/target/debug/rustfs" "$TEST_BINARY" +"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$DEBUG_DIR/rustfs" "$TEST_BINARY" cp "$LISTING_TMP" "$RUN_DIR/listing.json" export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}" +export RUSTFS_HEAL_CHAOS_LOG_DIR="${RUSTFS_HEAL_CHAOS_LOG_DIR:-$RUSTFS_E2E_LOG_DIR}" mkdir -p "$RUSTFS_E2E_LOG_DIR" -JUNIT_PATH="$ROOT/target/nextest/$PROFILE/junit.xml" +JUNIT_PATH="$TARGET_DIR/nextest/$PROFILE/junit.xml" rm -f "$JUNIT_PATH" set +e NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ diff --git a/scripts/run_scanner_heal_g09_upgrade_evidence.sh b/scripts/run_scanner_heal_g09_upgrade_evidence.sh new file mode 100755 index 000000000..6b348d6d1 --- /dev/null +++ b/scripts/run_scanner_heal_g09_upgrade_evidence.sh @@ -0,0 +1,654 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" +SOURCE_REPOSITORY="${RUSTFS_UPGRADE_SOURCE_REPOSITORY:-${UPGRADE_SOURCE_REPOSITORY:-rustfs/rustfs}}" +SOURCE_VERSION="${RUSTFS_UPGRADE_SOURCE_VERSION:-${UPGRADE_SOURCE_VERSION:-1.0.0-rc.5}}" +SOURCE_ASSET="${RUSTFS_UPGRADE_SOURCE_ASSET:-${UPGRADE_SOURCE_ASSET:-rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip}}" +SOURCE_SHA256="${RUSTFS_UPGRADE_SOURCE_SHA256:-${UPGRADE_SOURCE_SHA256:-3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d}}" +MIN_FREE_KIB="${RUSTFS_G09_MIN_FREE_KIB:-6291456}" + +RUN_DIR="" +SOURCE_DIR="" +SOURCE_BINARY="${RUSTFS_UPGRADE_SOURCE_BINARY:-}" +TEST_SELECTION="all" +PLAN_ONLY=0 +ALLOW_DIRTY=0 +SKIP_BUILD=0 +SKIP_DOWNLOAD=0 +VERBOSE=0 + +usage() { + cat <<'USAGE' +Usage: scripts/run_scanner_heal_g09_upgrade_evidence.sh [OPTIONS] + +Build the current checkout, run the Scanner/Heal G09 upgrade compatibility +lanes, validate the raw mixed-version/rollback evidence artifacts, and write a +bundle-ready G09 release-evidence descriptor for full runs. + +Options: + --run-dir DIR New evidence directory (default: target/scanner-heal-g09-evidence/TIMESTAMP) + --out-dir DIR Alias for --run-dir + --source-dir DIR Cache directory for the pinned previous release binary + --source-binary BIN Use an existing previous-release rustfs binary + --version VERSION Previous release tag (default: 1.0.0-rc.5) + --asset NAME Previous release asset zip name + --sha256 HEX Expected SHA-256 for the previous release asset + --repository OWNER/REPO + GitHub repository used to download the release asset (default: rustfs/rustfs) + --test NAME all, mixed-version, or rollback (default: all) + --allow-dirty Allow tracked source changes while collecting evidence + --skip-build Reuse an existing target/debug/rustfs binary + --skip-download Reuse SOURCE_DIR/rustfs instead of downloading the previous release + --plan-only Print the resolved plan without building or running tests + --dry-run Validate configuration and print the commands without running them + --self-test Run lightweight CLI and artifact-validator checks + --verbose Stream command output instead of storing it under the run directory + -h, --help Show this help + +The default pinned release asset is Linux x86_64. Use --source-binary when +running against a custom previous-release binary on another platform. The +script requires at least 6 GiB free by default; override +RUSTFS_G09_MIN_FREE_KIB only for a deliberately smaller diagnostic run. + +Environment overrides: + RUSTFS_SCANNER_HEAL_G09_OUTPUT_ROOT + RUSTFS_UPGRADE_SOURCE_DIR + RUSTFS_UPGRADE_SOURCE_BINARY + RUSTFS_UPGRADE_SOURCE_REPOSITORY / UPGRADE_SOURCE_REPOSITORY + RUSTFS_UPGRADE_SOURCE_VERSION / UPGRADE_SOURCE_VERSION + RUSTFS_UPGRADE_SOURCE_ASSET / UPGRADE_SOURCE_ASSET + RUSTFS_UPGRADE_SOURCE_SHA256 / UPGRADE_SOURCE_SHA256 + +Per-case test environment set by the runner: + RUSTFS_UPGRADE_SOURCE_BINARY + RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR + RUSTFS_E2E_LOG_DIR + +Required output files: + mixed-version-upgrade/G09-mixed_version_reader_evidence.json + mixed-version-upgrade/G09-mixed_version_writer_evidence.json + bucket-config-rollback/G09-rollback_payload_evidence.json +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_value() { + local option="$1" + local count="$2" + if [[ "$count" -lt 2 ]]; then + die "missing value for $option" + fi +} + +case_names() { + case "$TEST_SELECTION" in + all) + printf '%s\n' mixed-version rollback + ;; + mixed-version|rollback) + printf '%s\n' "$TEST_SELECTION" + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +validate_test_selection() { + case "$TEST_SELECTION" in + all|mixed-version|rollback) + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +artifact_for() { + case "$1" in + mixed-version) + echo "mixed-version-upgrade" + ;; + rollback) + echo "bucket-config-rollback" + ;; + *) + die "unknown G09 case: $1" + ;; + esac +} + +test_filter_for() { + case "$1" in + mixed-version) + echo "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts" + ;; + rollback) + echo "upgrade_compatibility_test::rollback_to_previous_release_reads_current_bucket_metadata" + ;; + *) + die "unknown G09 case: $1" + ;; + esac +} + +normalize_path() { + local path="$1" + if [[ "$path" == /* ]]; then + echo "$path" + else + echo "$ROOT/$path" + fi +} + +cargo_target_dir() { + if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then + normalize_path "$CARGO_TARGET_DIR" + else + echo "$ROOT/target" + fi +} + +write_rustfs_features_stamp() { + local target_dir + target_dir="$(cargo_target_dir)" + mkdir -p "$target_dir/debug" + : > "$target_dir/debug/rustfs.features" +} + +ensure_default_asset_platform() { + if [[ -n "$SOURCE_BINARY" ]]; then + return + fi + local os arch + os="$(uname -s)" + arch="$(uname -m)" + if [[ "$os" != "Linux" || ( "$arch" != "x86_64" && "$arch" != "amd64" ) ]]; then + echo "default previous-release asset requires Linux x86_64; pass --source-binary for this platform" >&2 + exit 2 + fi +} + +verify_sha256() { + local archive="$1" + if command -v sha256sum >/dev/null 2>&1 && [[ "$(sha256sum --help 2>&1)" == *"--check"* ]]; then + printf '%s %s\n' "$SOURCE_SHA256" "$archive" | sha256sum --check --strict + elif command -v shasum >/dev/null 2>&1; then + printf '%s %s\n' "$SOURCE_SHA256" "$archive" | shasum -a 256 --check + else + echo "sha256sum or shasum is required to verify $SOURCE_ASSET" >&2 + exit 1 + fi +} + +validate_source_sha256() { + [[ "$SOURCE_SHA256" =~ ^[0-9a-f]{64}$ ]] || die "--sha256 must be a 64-character lowercase hex digest" +} + +check_empty_case_dir() { + local dir="$1" + if [[ -d "$dir" ]] && find "$dir" -mindepth 1 -print -quit | grep -q .; then + die "evidence case directory is not empty: $dir" + fi +} + +ensure_min_free_space() { + local path="$1" + local available + mkdir -p "$path" + available="$(df -Pk "$path" | awk 'NR == 2 { print $4 }')" + if [[ -z "$available" ]]; then + echo "could not determine free space for $path" >&2 + exit 1 + fi + if (( available < MIN_FREE_KIB )); then + echo "insufficient free space for G09 evidence run at $path: need ${MIN_FREE_KIB} KiB, found ${available} KiB" >&2 + exit 1 + fi +} + +resolve_source_binary() { + if [[ -n "$SOURCE_BINARY" ]]; then + SOURCE_BINARY="$(normalize_path "$SOURCE_BINARY")" + test -x "$SOURCE_BINARY" + echo "$SOURCE_BINARY" + return + fi + + ensure_default_asset_platform + if [[ -z "$SOURCE_DIR" ]]; then + SOURCE_DIR="$ROOT/target/scanner-heal-g09-source/$SOURCE_VERSION" + else + SOURCE_DIR="$(normalize_path "$SOURCE_DIR")" + fi + + local binary="$SOURCE_DIR/rustfs" + if [[ -x "$binary" ]]; then + echo "$binary" + return + fi + if [[ "$SKIP_DOWNLOAD" == 1 ]]; then + die "previous release binary does not exist or is not executable: $binary" + fi + + mkdir -p "$SOURCE_DIR" + local archive="$SOURCE_DIR/$SOURCE_ASSET" + curl --fail --location --retry 3 --output "$archive" \ + "https://github.com/$SOURCE_REPOSITORY/releases/download/$SOURCE_VERSION/$SOURCE_ASSET" + verify_sha256 "$archive" >&2 + unzip -q "$archive" -d "$SOURCE_DIR" + chmod +x "$binary" + test -x "$binary" + echo "$binary" +} + +run_logged() { + local label="$1" + shift + local log="$RUN_DIR/logs/$label.log" + mkdir -p "$(dirname "$log")" + if [[ "$VERBOSE" == 1 ]]; then + "$@" + return + fi + if ! "$@" >"$log" 2>&1; then + echo "$label failed; log: $log" >&2 + tail -80 "$log" >&2 || true + return 1 + fi + echo "PASS: $label" +} + +utc_now() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +validate_artifacts() { + local source_revision="$1" + "$PYTHON_BIN" - "$RUN_DIR" "$source_revision" "$TEST_SELECTION" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +source_revision = sys.argv[2] +selection = sys.argv[3] + +expected = { + "mixed-version": [ + ("mixed-version-upgrade/G09-mixed_version_reader_evidence.json", + "mixed_version_reader_evidence", "mixed-version-reader"), + ("mixed-version-upgrade/G09-mixed_version_writer_evidence.json", + "mixed_version_writer_evidence", "mixed-version-writer"), + ], + "rollback": [ + ("bucket-config-rollback/G09-rollback_payload_evidence.json", + "rollback_payload_evidence", "rollback-payload"), + ], +} + +if selection != "all": + expected = {selection: expected[selection]} + +for _, artifacts in expected.items(): + for relative, field, role in artifacts: + path = run_dir / relative + if not path.is_file(): + raise SystemExit(f"missing G09 evidence artifact: {relative}") + evidence = json.loads(path.read_text()) + if evidence.get("schema") != 1: + raise SystemExit(f"{relative}: expected schema 1") + if evidence.get("evidence_type") != "measured": + raise SystemExit(f"{relative}: expected measured evidence") + if evidence.get("artifact_kind") != "upgrade-compatibility-e2e": + raise SystemExit(f"{relative}: unexpected artifact kind") + if evidence.get("gate") != "G09": + raise SystemExit(f"{relative}: unexpected gate") + if evidence.get("field") != field: + raise SystemExit(f"{relative}: expected field {field}") + if evidence.get("mixed_version_role") != role: + raise SystemExit(f"{relative}: expected role {role}") + if evidence.get("current_revision") != source_revision: + raise SystemExit(f"{relative}: current revision does not match this checkout") + versions = evidence.get("versions") + if not isinstance(versions, list) or len(versions) != 2: + raise SystemExit(f"{relative}: versions must contain previous and current revisions") + if versions[0] == versions[1]: + raise SystemExit(f"{relative}: previous and current revisions must differ") + if field == "rollback_payload_evidence" and evidence.get("rollback_payload_replayed") is not True: + raise SystemExit(f"{relative}: rollback payload was not marked replayed") + +print("PASS: G09 raw evidence artifacts verified") +PY +} + +write_g09_release_descriptor() { + local source_revision="$1" + local started_at="$2" + local finished_at="$3" + "$PYTHON_BIN" - "$ROOT" "$RUN_DIR" "$source_revision" "$started_at" "$finished_at" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +run_dir = pathlib.Path(sys.argv[2]) +source_revision = sys.argv[3] +started_at = sys.argv[4] +finished_at = sys.argv[5] +descriptor = run_dir / "release-bundle-g09.json" +registry = json.loads((root / ".config/scanner-heal-required-tests.json").read_text()) +requirements = {item["gate"]: item for item in registry["release_requirements"]} +artifacts = { + "mixed_version_reader_evidence": run_dir / "mixed-version-upgrade" / "G09-mixed_version_reader_evidence.json", + "mixed_version_writer_evidence": run_dir / "mixed-version-upgrade" / "G09-mixed_version_writer_evidence.json", + "rollback_payload_evidence": run_dir / "bucket-config-rollback" / "G09-rollback_payload_evidence.json", +} + +def digest(path: pathlib.Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + +def relative_to_descriptor(path: pathlib.Path) -> str: + return path.resolve(strict=True).relative_to(descriptor.parent.resolve()).as_posix() + +fields = {} +for field, artifact in artifacts.items(): + payload = json.loads(artifact.read_text()) + if payload.get("source_revision") != source_revision or payload.get("current_revision") != source_revision: + raise SystemExit(f"{field}: source revision does not match this checkout") + evidence = { + "artifact": relative_to_descriptor(artifact), + "sha256": digest(artifact), + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": payload["run_id"], + "measurement_window_id": payload["measurement_window_id"], + "started_at": started_at, + "finished_at": finished_at, + "command": ["scripts/run_scanner_heal_g09_upgrade_evidence.sh", "--test", "all"], + "artifact_format": "json", + "summary": f"Measured Scanner/Heal G09 {field} from the pinned previous release binary and current PR binary.", + "versions": payload["versions"], + "mixed_version_role": payload["mixed_version_role"], + "mixed_version_cases": payload["mixed_version_cases"], + } + if field == "rollback_payload_evidence": + evidence["rollback_payload_replayed"] = payload.get("rollback_payload_replayed") + fields[field] = evidence + +descriptor.write_text(json.dumps({ + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": { + "G09": { + "status": "pass", + "lane": requirements["G09"]["lane"], + "evidence_type": "measured", + "evidence_fields": fields, + }, + }, +}, indent=2, sort_keys=True) + "\n") +print(descriptor) +PY +} + +run_self_test() { + local tmp + tmp="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-g09-evidence-self-test.XXXXXX")" + trap "rm -rf '$tmp'" EXIT + + local plan + plan="$("$0" --plan-only --run-dir "$tmp/run" --source-binary "$tmp/rustfs-prev" --test all)" + [[ "$plan" == *"tests=mixed-version rollback"* ]] + [[ "$plan" == *"run_dir=$tmp/run"* ]] + [[ "$(CARGO_TARGET_DIR=relative-target "$0" --plan-only --run-dir "$tmp/run" --source-binary "$tmp/rustfs-prev")" == *"target_dir=$ROOT/relative-target"* ]] + + if "$0" --plan-only --test not-a-case >/dev/null 2>&1; then + echo "self-test failed: invalid test selection was accepted" >&2 + return 1 + fi + if "$0" --dry-run --sha256 bad >/dev/null 2>&1; then + echo "self-test failed: invalid SHA-256 was accepted" >&2 + return 1 + fi + mkdir -p "$tmp/nonempty/mixed-version-upgrade" + touch "$tmp/nonempty/mixed-version-upgrade/existing.json" + if "$0" --dry-run --run-dir "$tmp/nonempty" --sha256 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa >/dev/null 2>&1; then + echo "self-test failed: non-empty evidence directory was accepted" >&2 + return 1 + fi + + mkdir -p "$tmp/run/mixed-version-upgrade" "$tmp/run/bucket-config-rollback" + mkdir -p "$tmp/source" + local archive checksum checksum_output + archive="$tmp/source/$SOURCE_ASSET" + printf 'not a real archive\n' >"$archive" + if command -v shasum >/dev/null 2>&1; then + checksum="$(shasum -a 256 "$archive" | awk '{print $1}')" + else + checksum="$(sha256sum "$archive" | awk '{print $1}')" + fi + checksum_output="$(SOURCE_SHA256="$checksum" verify_sha256 "$archive")" + [[ "$checksum_output" == *"OK"* ]] + + local current previous + current="$(git rev-parse HEAD)" + previous="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + cat >"$tmp/run/mixed-version-upgrade/G09-mixed_version_reader_evidence.json" <"$tmp/run/mixed-version-upgrade/G09-mixed_version_writer_evidence.json" <"$tmp/run/bucket-config-rollback/G09-rollback_payload_evidence.json" </dev/null + local descriptor + descriptor="$(RUN_DIR="$tmp/run" write_g09_release_descriptor "$current" "$(utc_now)" "$(utc_now)")" + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$descriptor" G09 >/dev/null +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --run-dir|--out-dir) + require_value "$1" "$#" + RUN_DIR="$2" + shift 2 + ;; + --source-dir) + require_value "$1" "$#" + SOURCE_DIR="$2" + shift 2 + ;; + --source-binary) + require_value "$1" "$#" + SOURCE_BINARY="$2" + shift 2 + ;; + --version) + require_value "$1" "$#" + SOURCE_VERSION="$2" + shift 2 + ;; + --asset) + require_value "$1" "$#" + SOURCE_ASSET="$2" + shift 2 + ;; + --sha256) + require_value "$1" "$#" + SOURCE_SHA256="$2" + shift 2 + ;; + --repository) + require_value "$1" "$#" + SOURCE_REPOSITORY="$2" + shift 2 + ;; + --test) + require_value "$1" "$#" + TEST_SELECTION="$2" + shift 2 + ;; + --allow-dirty) + ALLOW_DIRTY=1 + shift + ;; + --skip-build) + SKIP_BUILD=1 + shift + ;; + --skip-download) + SKIP_DOWNLOAD=1 + shift + ;; + --plan-only|--dry-run) + PLAN_ONLY=1 + shift + ;; + --self-test) + run_self_test + exit $? + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +validate_test_selection +CASES=() +while IFS= read -r case_name; do + CASES+=("$case_name") +done < <(case_names) +if [[ -z "$RUN_DIR" ]]; then + RUN_DIR="$ROOT/target/scanner-heal-g09-evidence/$(date -u +%Y%m%dT%H%M%SZ)" +else + RUN_DIR="$(normalize_path "$RUN_DIR")" +fi +if [[ -n "$SOURCE_DIR" ]]; then + SOURCE_DIR="$(normalize_path "$SOURCE_DIR")" +fi + +validate_source_sha256 +for case_name in "${CASES[@]}"; do + check_empty_case_dir "$RUN_DIR/$(artifact_for "$case_name")" +done + +if [[ "$PLAN_ONLY" == 1 ]]; then + echo "run_dir=$RUN_DIR" + echo "out_dir=$RUN_DIR" + echo "tests=${CASES[*]}" + echo "source_repository=$SOURCE_REPOSITORY" + echo "source_version=$SOURCE_VERSION" + echo "min_free_kib=$MIN_FREE_KIB" + if [[ -n "$SOURCE_BINARY" ]]; then + echo "source_binary=$(normalize_path "$SOURCE_BINARY")" + else + if [[ -z "$SOURCE_DIR" ]]; then + echo "source_dir=$ROOT/target/scanner-heal-g09-source/$SOURCE_VERSION" + else + echo "source_dir=$SOURCE_DIR" + fi + echo "source_asset=$SOURCE_ASSET" + echo "download_url=https://github.com/$SOURCE_REPOSITORY/releases/download/$SOURCE_VERSION/$SOURCE_ASSET" + fi + target_dir="$(cargo_target_dir)" + echo "target_dir=$target_dir" + echo "current_binary=$target_dir/debug/rustfs" + echo "test_filters:" + for case_name in "${CASES[@]}"; do + echo " $(artifact_for "$case_name"): $(test_filter_for "$case_name")" + done + echo "required_artifacts:" + for case_name in "${CASES[@]}"; do + artifact="$(artifact_for "$case_name")" + case "$case_name" in + mixed-version) + echo " $RUN_DIR/$artifact/G09-mixed_version_reader_evidence.json" + echo " $RUN_DIR/$artifact/G09-mixed_version_writer_evidence.json" + ;; + rollback) + echo " $RUN_DIR/$artifact/G09-rollback_payload_evidence.json" + ;; + esac + done + exit 0 +fi + +cd "$ROOT" +if [[ "$ALLOW_DIRTY" != 1 && -n "$(git status --porcelain --untracked-files=no)" ]]; then + echo "commit tracked source changes before creating release evidence, or pass --allow-dirty for local diagnostics" >&2 + exit 1 +fi +if [[ -e "$RUN_DIR" ]]; then + echo "evidence run directory already exists: $RUN_DIR" >&2 + exit 1 +fi +mkdir -p "$RUN_DIR/logs" +if [[ -n "${TMPDIR:-}" ]]; then + mkdir -p "$TMPDIR" + ensure_min_free_space "$TMPDIR" +fi +ensure_min_free_space "$RUN_DIR" + +RUN_STARTED_AT="$(utc_now)" +SOURCE_BINARY="$(resolve_source_binary)" +export RUSTFS_UPGRADE_SOURCE_BINARY="$SOURCE_BINARY" +export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/server-logs}" +mkdir -p "$RUSTFS_E2E_LOG_DIR" + +if [[ "$SKIP_BUILD" != 1 ]]; then + run_logged build-current cargo build --locked -p rustfs --bin rustfs + write_rustfs_features_stamp +fi + +SOURCE_REVISION="$(git rev-parse HEAD)" +printf '%s\n' "$SOURCE_REVISION" >"$RUN_DIR/source-revision.txt" +printf '%s\n' "$SOURCE_VERSION" >"$RUN_DIR/previous-release-version.txt" + +for case_name in "${CASES[@]}"; do + artifact="$(artifact_for "$case_name")" + test_filter="$(test_filter_for "$case_name")" + evidence_dir="$RUN_DIR/$artifact" + mkdir -p "$evidence_dir" + run_logged "$case_name" env \ + NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ + HTTP_PROXY= \ + HTTPS_PROXY= \ + RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR="$evidence_dir" \ + cargo test --locked -p e2e_test "$test_filter" -- --ignored --exact --nocapture +done + +validate_artifacts "$SOURCE_REVISION" +if [[ "$TEST_SELECTION" == "all" ]]; then + RUN_FINISHED_AT="$(utc_now)" + DESCRIPTOR="$(write_g09_release_descriptor "$SOURCE_REVISION" "$RUN_STARTED_AT" "$RUN_FINISHED_AT")" + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$DESCRIPTOR" G09 + echo "Scanner/Heal G09 release descriptor verified: $DESCRIPTOR" +fi +echo "Scanner/Heal G09 evidence verified: $RUN_DIR" diff --git a/scripts/run_scanner_heal_g14_multiset_evidence.py b/scripts/run_scanner_heal_g14_multiset_evidence.py new file mode 100755 index 000000000..e2b2bd23a --- /dev/null +++ b/scripts/run_scanner_heal_g14_multiset_evidence.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal G14 multi-set/multi-pool release evidence. + +The input proof must come from one same-window distributed measurement that +already observed EC8+4, at least two sets, at least two pools, and distributed +segment invalidation. This script packages that proof into the release-bundle +field shape enforced by check_test_wiring.py. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +from scanner_abba import digest, read_json, require, write_json + +ROOT = Path(__file__).resolve().parents[1] +G14_FIELDS = ( + "same_window_field_evidence", + "ec8_4_evidence", + "multi_set_evidence", + "multi_pool_evidence", + "distributed_segment_invalidation_evidence", +) + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def iso_from_epoch(value: Any, name: str) -> str: + require(type(value) in (int, float) and value > 0, f"invalid {name}") + return datetime.fromtimestamp(float(value), timezone.utc).isoformat().replace("+00:00", "Z") + + +def positive_int(value: Any, name: str, minimum: int = 1) -> int: + require(type(value) is int and value >= minimum, f"invalid {name}") + return value + + +def reject_non_measured_markers(payload: dict[str, Any], label: str) -> None: + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(payload.get(marker) is not True, f"{label} is {marker}") + + +def parse_case_dir_arg(value: str) -> tuple[str | None, Path]: + if "=" in value: + case_id, raw_path = value.split("=", 1) + require(bool(case_id.strip()), "case directory case id is empty") + return case_id.strip(), Path(raw_path).expanduser().resolve() + return None, Path(value).expanduser().resolve() + + +def proof_case_artifact_path(proof_path: Path, sample: dict[str, Any], index: int) -> Path: + raw_artifact = sample.get("artifact") + require(isinstance(raw_artifact, str) and raw_artifact.strip(), f"G14 case evidence {index} missing artifact") + artifact = Path(raw_artifact) + require(not artifact.is_absolute() and ".." not in artifact.parts, + f"G14 case evidence {index} artifact path escapes proof directory") + resolved = (proof_path.parent / artifact).resolve() + require(resolved.is_relative_to(proof_path.parent.resolve()), + f"G14 case evidence {index} artifact path escapes proof directory") + require(resolved.is_file(), f"G14 case evidence {index} artifact is missing") + require(resolved.stat().st_size > 0, f"G14 case evidence {index} artifact is empty") + return resolved + + +def load_proof(path: Path, source_revision: str) -> dict[str, Any]: + proof = read_json(path) + reject_non_measured_markers(proof, "G14 proof") + require(proof.get("schema") == 1, "unsupported G14 proof schema") + require(proof.get("evidence_type") == "measured", "G14 proof must be measured") + require(proof.get("source_revision") == source_revision, "G14 proof source revision mismatch") + require(isinstance(proof.get("run_id"), str) and len(proof["run_id"]) >= 8, "missing G14 proof run_id") + require(isinstance(proof.get("measurement_window_id"), str) and len(proof["measurement_window_id"]) >= 8, + "missing G14 measurement window") + require(proof["run_id"] != proof["measurement_window_id"], "G14 run/window identities must differ") + require(isinstance(proof.get("command"), list) and proof["command"], "missing G14 command provenance") + topology = proof.get("topology") + require(isinstance(topology, dict), "G14 proof missing topology") + require(topology.get("erasure") == "EC8+4", "G14 proof must record EC8+4") + positive_int(topology.get("nodes"), "topology.nodes", 3) + positive_int(topology.get("drives_per_node"), "topology.drives_per_node", 4) + positive_int(proof.get("sets"), "sets", 2) + positive_int(proof.get("pools"), "pools", 2) + require(proof.get("invalidation_domain") == "distributed-ec", "G14 proof must be distributed EC invalidation") + require(proof.get("distributed_ec_invalidation") is True, "G14 proof missing peer invalidation") + positive_int(proof.get("peer_count"), "peer_count", 3) + require(proof.get("same_window_remote_proof") is True, "G14 proof missing same-window remote proof") + require(proof.get("all_peers_bound_to_generation_window") is True, + "G14 proof missing peer generation-window binding") + samples = proof.get("case_evidence") + require(isinstance(samples, list) and samples, "G14 proof must reference measured case evidence") + for index, sample in enumerate(samples): + require(isinstance(sample, dict), f"G14 case evidence {index} must be an object") + require(isinstance(sample.get("case"), str) and sample["case"].strip(), f"G14 case evidence {index} missing case") + require(isinstance(sample.get("sha256"), str) and re.fullmatch(r"[0-9a-f]{64}", sample["sha256"]), + f"G14 case evidence {index} missing sha256") + artifact = proof_case_artifact_path(path, sample, index) + require(digest(artifact) == sample["sha256"], f"G14 case evidence {index} artifact hash mismatch") + require(sample.get("source_revision") == source_revision, + f"G14 case evidence {index} source revision mismatch") + require(sample.get("measurement_window_id") == proof["measurement_window_id"], + f"G14 case evidence {index} measurement window mismatch") + artifact_payload = read_json(artifact) + require(isinstance(artifact_payload, dict), f"G14 case evidence {index} artifact must be a JSON object") + reject_non_measured_markers(artifact_payload, f"G14 case evidence {index} artifact") + require(artifact_payload.get("case") == sample["case"], f"G14 case evidence {index} artifact case mismatch") + if "source_revision" in artifact_payload: + require(artifact_payload["source_revision"] == source_revision, + f"G14 case evidence {index} artifact source revision mismatch") + if "measurement_window_id" in artifact_payload: + require(artifact_payload["measurement_window_id"] == proof["measurement_window_id"], + f"G14 case evidence {index} artifact measurement window mismatch") + return proof + + +def load_case_directory(raw_value: str, source_revision: str) -> dict[str, Any]: + expected_case, directory = parse_case_dir_arg(raw_value) + require(directory.is_dir(), f"G14 case directory is missing: {directory}") + run = read_json(directory / "run.json") + execution = read_json(directory / "execution.json") + require(isinstance(run, dict), "G14 case run must be a JSON object") + require(isinstance(execution, dict), "G14 case execution must be a JSON object") + reject_non_measured_markers(run, "G14 case run") + reject_non_measured_markers(execution, "G14 case execution") + require(run.get("schema") == 1, "G14 case run schema mismatch") + require(isinstance(run.get("run_id"), str) and re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), + "invalid G14 case run id") + require(run.get("source_revision") == source_revision, "G14 case source revision mismatch") + require(execution.get("run_id") == run["run_id"], "G14 case execution belongs to another run") + require(execution.get("exit_code") == 0, "G14 case execution did not pass") + artifacts = execution.get("artifacts") + require(isinstance(artifacts, dict), "G14 case execution missing artifacts") + started_at = iso_from_epoch(run.get("started_at"), "case started_at") + finished_at = iso_from_epoch(execution.get("finished_at"), "case finished_at") + + oracle_items = [] + for name, expected_sha in sorted(artifacts.items()): + if not name.endswith(".json") or name in {"run.json", "execution.json", "release-status.json"}: + continue + path = directory / name + if not path.is_file(): + continue + require(isinstance(expected_sha, str) and re.fullmatch(r"[0-9a-f]{64}", expected_sha), + f"G14 case artifact {name} has an invalid hash") + require(digest(path) == expected_sha, f"G14 case artifact hash mismatch: {name}") + item = read_json(path) + if item.get("schema") == 1 and item.get("case") and item.get("run_id") == run["run_id"]: + oracle_items.append((name, path, expected_sha, item)) + require(len(oracle_items) == 1, "G14 case directory must contain exactly one case oracle") + name, path, expected_sha, oracle = oracle_items[0] + if expected_case is not None: + require(oracle.get("case") == expected_case, "G14 case directory case id mismatch") + require(oracle.get("source_revision") == source_revision, "G14 oracle source revision mismatch") + reject_non_measured_markers(oracle, "G14 oracle") + require(oracle.get("evidence") in {"process-restart", "process-crash-restart"}, "G14 oracle has wrong evidence type") + topology = oracle.get("topology") + require(isinstance(topology, dict), "G14 oracle missing topology") + positive_int(topology.get("nodes"), "oracle topology.nodes", 3) + positive_int(topology.get("drives_per_node"), "oracle topology.drives_per_node", 4) + require(oracle.get("erasure_set_drive_count") == 12, "G14 oracle must use EC8+4 set width") + positive_int(oracle.get("sets"), "oracle sets", 1) + positive_int(oracle.get("pools"), "oracle pools", 1) + if oracle.get("sets", 1) > 1 or oracle.get("pools", 1) > 1: + require(oracle.get("distributed_ec_invalidation") is True, "G14 oracle missing distributed invalidation") + positive_int(oracle.get("peer_count"), "oracle peer_count", 3) + require(oracle.get("same_window_remote_proof") is True, "G14 oracle missing same-window remote proof") + require(oracle.get("all_peers_bound_to_generation_window") is True, + "G14 oracle missing peer generation-window binding") + return { + "case": oracle["case"], + "artifact_name": name, + "artifact_path": path, + "sha256": expected_sha, + "run_id": run["run_id"], + "started_at": started_at, + "finished_at": finished_at, + "oracle": oracle, + } + + +def copy_case_artifacts(out_dir: Path, records: list[dict[str, Any]], window_id: str, + source_revision: str) -> list[dict[str, Any]]: + case_dir = out_dir / "artifacts" / "cases" + case_dir.mkdir(parents=True, exist_ok=True) + copied = [] + for record in records: + target = case_dir / record["artifact_name"] + if target.exists(): + target = case_dir / f"{record['case']}-{record['artifact_name']}" + shutil.copyfile(record["artifact_path"], target) + copied.append({ + "case": record["case"], + "artifact": target.relative_to(out_dir).as_posix(), + "sha256": digest(target), + "source_revision": source_revision, + "measurement_window_id": window_id, + }) + return copied + + +def copy_proof_case_artifacts(out_dir: Path, proof_path: Path, samples: list[dict[str, Any]], window_id: str, + source_revision: str) -> list[dict[str, Any]]: + case_dir = out_dir / "artifacts" / "cases" + case_dir.mkdir(parents=True, exist_ok=True) + copied = [] + for index, sample in enumerate(samples): + source = proof_case_artifact_path(proof_path, sample, index) + safe_case = re.sub(r"[^A-Za-z0-9._-]", "-", sample["case"]) + target = case_dir / f"{index:02d}-{safe_case}{source.suffix or '.json'}" + require(not target.exists(), f"G14 case evidence {index} destination already exists") + shutil.copyfile(source, target) + copied.append({ + "case": sample["case"], + "artifact": target.relative_to(out_dir).as_posix(), + "sha256": digest(target), + "source_revision": source_revision, + "measurement_window_id": window_id, + }) + return copied + + +def proof_from_case_directories(raw_values: list[str], out_dir: Path, source_revision: str) -> dict[str, Any]: + require(raw_values, "missing G14 case directories") + records = [load_case_directory(value, source_revision) for value in raw_values] + covering = [ + record for record in records + if record["oracle"].get("erasure_set_drive_count") == 12 + and record["oracle"].get("sets", 0) >= 2 + and record["oracle"].get("pools", 0) >= 2 + and record["oracle"].get("distributed_ec_invalidation") is True + ] + require(covering, "G14 case evidence must include one same-window EC8+4 multi-set/multi-pool proof") + selected = covering[0] + oracle = selected["oracle"] + window_id = f"g14-case-window-{selected['run_id']}" + return { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": selected["run_id"], + "measurement_window_id": window_id, + "started_at": selected["started_at"], + "finished_at": selected["finished_at"], + "command": ["scripts/run_scanner_heal_g14_multiset_evidence.py", "--case-dir", ""], + "summary": "Measured G14 descriptor assembled from Scanner/Heal e2e case evidence.", + "topology": { + "erasure": "EC8+4", + "nodes": oracle["topology"]["nodes"], + "drives_per_node": oracle["topology"]["drives_per_node"], + }, + "sets": oracle["sets"], + "pools": oracle["pools"], + "invalidation_domain": "distributed-ec", + "distributed_ec_invalidation": oracle["distributed_ec_invalidation"], + "peer_count": oracle["peer_count"], + "same_window_remote_proof": oracle["same_window_remote_proof"], + "all_peers_bound_to_generation_window": oracle["all_peers_bound_to_generation_window"], + "case_evidence": copy_case_artifacts(out_dir, [selected], window_id, source_revision), + } + + +def write_field(out_dir: Path, field: str, proof: dict[str, Any], source_revision: str) -> dict[str, Any]: + started_at = proof.get("started_at") or utc_now() + finished_at = proof.get("finished_at") or started_at + evidence: dict[str, Any] = { + "artifact": "", + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "started_at": started_at, + "finished_at": finished_at, + "command": proof["command"], + "artifact_format": "json", + "summary": proof.get("summary") or "Measured G14 same-window EC8+4 multi-set/multi-pool proof.", + "case_evidence": proof["case_evidence"], + } + if field == "same_window_field_evidence": + evidence["same_window_fields"] = [item for item in G14_FIELDS if item != field] + elif field == "ec8_4_evidence": + evidence["topology"] = proof["topology"] + elif field == "multi_set_evidence": + evidence["sets"] = proof["sets"] + elif field == "multi_pool_evidence": + evidence["pools"] = proof["pools"] + elif field == "distributed_segment_invalidation_evidence": + for key in ( + "invalidation_domain", + "distributed_ec_invalidation", + "peer_count", + "same_window_remote_proof", + "all_peers_bound_to_generation_window", + ): + evidence[key] = proof[key] + artifact = out_dir / "artifacts" / f"G14-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "gate": "G14", + "field": field, + "case_evidence": proof["case_evidence"], + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = digest(artifact) + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + out_dir.mkdir(parents=True) + if args.proof_json is not None: + proof_path = args.proof_json.resolve() + proof = load_proof(proof_path, source_revision) + proof["case_evidence"] = copy_proof_case_artifacts( + out_dir, + proof_path, + proof["case_evidence"], + proof["measurement_window_id"], + source_revision, + ) + else: + proof = proof_from_case_directories(args.case_dir, out_dir, source_revision) + fields = {field: write_field(out_dir, field, proof, source_revision) for field in G14_FIELDS} + descriptor = out_dir / "release-bundle-g14.json" + write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": { + "G14": { + "status": "pass", + "lane": "ec8-4-multiset", + "evidence_type": "measured", + "evidence_fields": fields, + }, + }, + }) + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + "G14", + ], cwd=ROOT) + return descriptor + + +def write_self_test_proof(root: Path, source_revision: str) -> Path: + window_id = "g14-self-test-window" + case_evidence = [] + for case in ( + "ec84-target-drive-restart", + "multi-set-distributed-invalidation", + "multi-pool-distributed-invalidation", + ): + artifact = root / f"{case}.json" + write_json(artifact, { + "schema": 1, + "case": case, + "evidence_type": "measured", + "source_revision": source_revision, + "measurement_window_id": window_id, + }) + case_evidence.append({ + "case": case, + "artifact": artifact.relative_to(root).as_posix(), + "sha256": digest(artifact), + "source_revision": source_revision, + "measurement_window_id": window_id, + }) + proof = root / "proof.json" + write_json(proof, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": "g14-self-test-run", + "measurement_window_id": window_id, + "started_at": "2026-09-09T00:00:00Z", + "finished_at": "2026-09-09T00:30:00Z", + "command": ["scripts/run_scanner_heal_g14_multiset_evidence.py", "--proof-json", "proof.json"], + "summary": "Measured parser self-test proof.", + "topology": {"erasure": "EC8+4", "nodes": 3, "drives_per_node": 4}, + "sets": 2, + "pools": 2, + "invalidation_domain": "distributed-ec", + "distributed_ec_invalidation": True, + "peer_count": 3, + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + "case_evidence": case_evidence, + }) + return proof + + +def write_self_test_case_dir(root: Path, source_revision: str, case: str, sets: int, pools: int) -> Path: + directory = root / case + directory.mkdir() + started = datetime(2026, 9, 9, 0, 0, 0, tzinfo=timezone.utc).timestamp() + finished = datetime(2026, 9, 9, 0, 30, 0, tzinfo=timezone.utc).timestamp() + run_id = ("a" if pools > 1 else "b") * 32 + write_json(directory / "run.json", { + "schema": 1, + "run_id": run_id, + "source_revision": source_revision, + "started_at": started, + }) + oracle = { + "schema": 1, + "case": case, + "evidence": "process-crash-restart" if pools > 1 else "process-restart", + "run_id": run_id, + "source_revision": source_revision, + "topology": {"nodes": 3, "drives_per_node": 12 if pools > 1 else 8}, + "erasure_set_drive_count": 12, + "sets": sets, + "pools": pools, + "distributed_ec_invalidation": True, + "peer_count": 3, + "same_window_remote_proof": True, + "all_peers_bound_to_generation_window": True, + } + oracle_name = f"{case}.json" + write_json(directory / oracle_name, oracle) + write_json(directory / "execution.json", { + "run_id": run_id, + "exit_code": 0, + "finished_at": finished, + "artifacts": {oracle_name: digest(directory / oracle_name)}, + }) + return directory + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = write_self_test_proof(root, source_revision) + descriptor = build_descriptor(parse_args([ + "--proof-json", str(proof), + "--out-dir", str(root / "out"), + ])) + require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = write_self_test_proof(root, source_revision) + data = read_json(proof) + data["pools"] = 1 + write_json(proof, data) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + require("pools" in str(err), "wrong self-test failure for single-pool proof") + else: + raise ValueError("self-test accepted single-pool G14 proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + multi_set = write_self_test_case_dir(root, source_revision, "background-target-restart-ec8-4-multi-set", 2, 1) + multi_pool = write_self_test_case_dir(root, source_revision, "background-target-crash-ec8-4-multi-pool", 3, 3) + descriptor = build_descriptor(parse_args([ + "--case-dir", f"background-target-restart-ec8-4-multi-set={multi_set}", + "--case-dir", f"background-target-crash-ec8-4-multi-pool={multi_pool}", + "--out-dir", str(root / "out"), + ])) + require(descriptor.is_file(), "self-test case-dir descriptor missing") + data = read_json(descriptor) + evidence = data["gates"]["G14"]["evidence_fields"]["same_window_field_evidence"] + artifact = descriptor.parent / evidence["artifact"] + case_evidence = read_json(artifact)["case_evidence"] + require(len(case_evidence) == 1, "self-test case-dir descriptor copied non-covering case evidence") + require(case_evidence[0]["case"] == "background-target-crash-ec8-4-multi-pool", + "self-test case-dir descriptor selected the wrong covering case") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + multi_set = write_self_test_case_dir(root, source_revision, "background-target-restart-ec8-4-multi-set", 2, 1) + try: + build_descriptor(parse_args([ + "--case-dir", f"background-target-restart-ec8-4-multi-set={multi_set}", + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("multi-set/multi-pool" in str(err), "wrong self-test failure for missing multi-pool case") + else: + raise ValueError("self-test accepted case evidence without multi-pool proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + multi_pool = write_self_test_case_dir(root, source_revision, "background-target-crash-ec8-4-multi-pool", 3, 3) + oracle_path = multi_pool / "background-target-crash-ec8-4-multi-pool.json" + oracle = read_json(oracle_path) + oracle["fixture_only"] = True + write_json(oracle_path, oracle) + execution = read_json(multi_pool / "execution.json") + execution["artifacts"][oracle_path.name] = digest(oracle_path) + write_json(multi_pool / "execution.json", execution) + try: + build_descriptor(parse_args([ + "--case-dir", f"background-target-crash-ec8-4-multi-pool={multi_pool}", + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("G14 oracle is fixture_only" in str(err), "wrong self-test failure for fixture oracle") + else: + raise ValueError("self-test accepted fixture-only G14 case oracle") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proof-json", type=Path) + parser.add_argument("--case-dir", action="append", default=[], + help="Measured e2e evidence run directory, optionally CASE=DIR; repeatable") + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if (args.proof_json is None) == (not args.case_dir): + parser.error("provide exactly one of --proof-json or --case-dir unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"G14 release descriptor verified: {descriptor}") + return 0 + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_legacy_rollback_evidence.py b/scripts/run_scanner_heal_legacy_rollback_evidence.py new file mode 100755 index 000000000..9186c172a --- /dev/null +++ b/scripts/run_scanner_heal_legacy_rollback_evidence.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal R-L legacy rollback release evidence. + +This producer consumes one operator-collected measured JSON proof. It packages +legacy source-conflict, migration-gap, and crash-safe source-retirement evidence +into the common release-bundle descriptor shape. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import check_test_wiring as wiring + +ROOT = Path(__file__).resolve().parents[1] +RL_FIELDS = ( + "legacy_source_conflict_evidence", + "migration_gap_evidence", + "crash_safe_source_retirement_evidence", +) + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def timestamp(value: Any, name: str) -> str: + wiring.require(isinstance(value, str) and value.strip(), f"missing {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + wiring.require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def measured_proof(path: Path, source_revision: str) -> dict[str, Any]: + proof = wiring.read_json(path) + wiring.require(proof.get("schema") == 1, "proof schema must be 1") + wiring.require(proof.get("evidence_type") == "measured", "proof must be measured") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(proof.get(marker) is not True, f"proof is {marker}") + wiring.require(proof.get("source_revision") == source_revision, "proof source revision mismatch") + started_at = timestamp(proof.get("started_at"), "proof.started_at") + finished_at = timestamp(proof.get("finished_at"), "proof.finished_at") + wiring.require( + datetime.fromisoformat(finished_at.replace("Z", "+00:00")) + >= datetime.fromisoformat(started_at.replace("Z", "+00:00")), + "proof timestamps are inverted", + ) + wiring.evidence_string(proof.get("run_id"), "proof.run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + wiring.evidence_string( + proof.get("measurement_window_id"), + "proof.measurement_window_id", + r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", + ) + wiring.require(proof["measurement_window_id"] != proof["run_id"], "proof must separate run/window identities") + versions = proof.get("versions") + wiring.require( + isinstance(versions, list) + and len(set(versions)) >= 2 + and all(isinstance(version, str) and re.fullmatch(r"[0-9a-f]{40}", version) for version in versions), + "proof requires two source revisions", + ) + wiring.require(source_revision in versions, "proof versions omit tested source revision") + return proof + + +def field_from_proof(proof: dict[str, Any], field: str) -> dict[str, Any]: + value = proof.get(field) + wiring.require(isinstance(value, dict), f"proof missing {field}") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(value.get(marker) is not True, f"{field} is {marker}") + if "evidence_type" in value: + wiring.require(value["evidence_type"] == "measured", f"{field} must be measured") + if "source_revision" in value: + wiring.require(value["source_revision"] == proof["source_revision"], f"{field} source revision mismatch") + if "run_id" in value: + wiring.require(value["run_id"] == proof["run_id"], f"{field} run_id mismatch") + if "measurement_window_id" in value: + wiring.require( + value["measurement_window_id"] == proof["measurement_window_id"], + f"{field} measurement window mismatch", + ) + evidence = { + **value, + "versions": proof["versions"], + "mixed_version_role": wiring.SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[("R-L", field)], + } + if "crash_points" not in evidence: + evidence["crash_points"] = proof.get("crash_points") + wiring.validate_release_bundle_domain_evidence("R-L", field, evidence) + return evidence + + +def write_field(out_dir: Path, common: dict[str, Any], field: str, field_evidence: dict[str, Any]) -> dict[str, Any]: + evidence = { + **field_evidence, + **common, + "evidence_type": "measured", + "summary": field_evidence.get("summary") or f"Measured Scanner/Heal R-L {field} evidence.", + } + artifact = out_dir / "artifacts" / f"R-L-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "gate": "R-L", + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + wiring.write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = wiring.digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + wiring.require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + proof = measured_proof(args.proof_json.resolve(), source_revision) + out_dir.mkdir(parents=True) + common = { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "started_at": timestamp(proof["started_at"], "proof.started_at"), + "finished_at": timestamp(proof["finished_at"], "proof.finished_at"), + "command": [ + "scripts/run_scanner_heal_legacy_rollback_evidence.py", + "--proof-json", + "", + ], + } + descriptor = out_dir / "release-bundle-legacy-rollback.json" + gates = { + "R-L": { + "status": "pass", + "lane": "mixed-version-rollback", + "evidence_type": "measured", + "evidence_fields": { + field: write_field(out_dir, common, field, field_from_proof(proof, field)) + for field in RL_FIELDS + }, + }, + } + wiring.write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + "R-L", + ], cwd=ROOT) + return descriptor + + +def write_self_test_proof(path: Path, source_revision: str) -> None: + now = datetime.now(timezone.utc).replace(microsecond=0) + proof = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": f"legacy-rollback-{source_revision[:12]}", + "measurement_window_id": f"legacy-rollback-window-{source_revision[:12]}", + "started_at": now.isoformat().replace("+00:00", "Z"), + "finished_at": now.isoformat().replace("+00:00", "Z"), + "versions": ["a" * 40, source_revision], + "crash_points": ["before-successor-manifest", "after-successor-before-retire"], + "legacy_source_conflict_evidence": { + "legacy_source_conflict_cases": list( + wiring.SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES["legacy_source_conflict_evidence"] + ), + "source_conflicts_rejected": True, + "takeover_identity_bound": True, + "legacy_checksum_gap_rejected": True, + }, + "migration_gap_evidence": { + "migration_gap_cases": list(wiring.SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES["migration_gap_evidence"]), + "migration_gap_closed": True, + "legacy_sources_fail_closed": True, + "prior_responsibilities_inherited": True, + }, + "crash_safe_source_retirement_evidence": { + "source_retirement_cases": list( + wiring.SCANNER_HEAL_RELEASE_RL_REQUIRED_CASES["crash_safe_source_retirement_evidence"] + ), + "source_retirement_is_crash_safe": True, + "old_source_retained_until_successor": True, + "recovered_pending_migration": True, + }, + } + wiring.write_json(path, proof) + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "legacy-rollback-proof.json" + write_self_test_proof(proof, source_revision) + descriptor = build_descriptor(parse_args([ + "--proof-json", str(proof), + "--out-dir", str(root / "out"), + ])) + wiring.require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "legacy-rollback-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["migration_gap_evidence"]["migration_gap_closed"] = False + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("migration_gap_closed" in str(err), "wrong self-test failure for R-L proof") + else: + raise ValueError("self-test accepted incomplete R-L proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "legacy-rollback-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["evidence_type"] = "synthetic" + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("measured" in str(err), "wrong self-test failure for synthetic proof") + else: + raise ValueError("self-test accepted synthetic proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "legacy-rollback-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["versions"] = ["not-a-source-revision".ljust(40, "x"), source_revision] + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("source revisions" in str(err), "wrong self-test failure for invalid version") + else: + raise ValueError("self-test accepted invalid source revision") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "legacy-rollback-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["finished_at"] = "2026-09-08T00:00:00Z" + payload["started_at"] = "2026-09-09T00:00:00Z" + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("timestamps are inverted" in str(err), "wrong self-test failure for inverted timestamps") + else: + raise ValueError("self-test accepted inverted timestamps") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "legacy-rollback-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["legacy_source_conflict_evidence"]["run_id"] = "legacy-rollback-different-run" + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("run_id mismatch" in str(err), "wrong self-test failure for nested run_id") + else: + raise ValueError("self-test accepted nested run_id mismatch") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proof-json", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.proof_json is None: + parser.error("--proof-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Legacy rollback release descriptor verified: {descriptor}") + return 0 + except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_linux_evidence_plan.py b/scripts/run_scanner_heal_linux_evidence_plan.py new file mode 100755 index 000000000..5c83b4082 --- /dev/null +++ b/scripts/run_scanner_heal_linux_evidence_plan.py @@ -0,0 +1,715 @@ +#!/usr/bin/env python3 +"""Plan the Scanner/Heal Linux release-evidence validation flow. + +The planner is intentionally not a release-evidence producer. It writes a +machine-readable execution manifest for the existing measured runners and can +run only their lightweight preflight checks. Long-running Linux, distributed, +mixed-version, ABBA, and profile lanes remain explicit operator actions. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +REGISTRY = ROOT / ".config" / "scanner-heal-required-tests.json" +DEFAULT_OUTPUT_ROOT = ROOT / "target" / "scanner-heal-linux-evidence-plan" +REQUIRED_GATES = { + "G01", + "G02", + "G03", + "G04", + "G05", + "G06", + "G07", + "G08", + "G09", + "G10", + "G11", + "G12", + "G13", + "G14", + "P1", + "P2", + "P3", + "P4", + "R-D", + "R-E", + "R-L", +} + + +def command(*parts: str) -> list[str]: + return list(parts) + + +def expected_outputs(*parts: str) -> list[str]: + return list(parts) + + +def git_output(*args: str) -> str: + return subprocess.check_output(command("git", *args), cwd=ROOT, text=True).strip() + + +def source_revision(explicit: str | None) -> str: + if explicit: + if len(explicit) != 40 or any(char not in "0123456789abcdef" for char in explicit): + raise ValueError("--source-revision must be a 40-character lowercase Git SHA") + return explicit + return git_output("rev-parse", "HEAD") + + +def load_registry() -> dict[str, Any]: + with REGISTRY.open() as stream: + registry = json.load(stream) + if registry.get("schema") != 2: + raise ValueError("Scanner/Heal release registry must use schema 2") + return registry + + +def validate_registry(registry: dict[str, Any]) -> None: + gates = {item["gate"] for item in registry.get("release_requirements", [])} + missing = sorted(REQUIRED_GATES - gates) + if missing: + raise ValueError(f"release registry is missing gates: {', '.join(missing)}") + lanes = registry.get("release_lanes") + if not isinstance(lanes, dict) or not lanes: + raise ValueError("release registry is missing release_lanes") + lane_gates = set() + for lane_name, lane in lanes.items(): + if not isinstance(lane, dict): + raise ValueError(f"release lane {lane_name} must be an object") + lane_gates.update(lane.get("gates", [])) + missing_from_lanes = sorted(gates - lane_gates) + if missing_from_lanes: + raise ValueError(f"release lanes do not cover gates: {', '.join(missing_from_lanes)}") + + +def existing_script_command(*parts: str) -> dict[str, Any]: + for part in parts: + if part.startswith("scripts/") and "$" not in part and not (ROOT / part).is_file(): + raise ValueError(f"missing planned script: {part}") + return {"command": command(*parts), "script": parts[0]} + + +def preflight_steps() -> list[dict[str, Any]]: + return [ + { + "id": "repo-source-clean", + "description": "Verify the checkout revision and tracked-source cleanliness before measured evidence.", + "commands": [ + {"command": command("git", "rev-parse", "HEAD")}, + {"command": command("git", "status", "--porcelain", "--untracked-files=no")}, + ], + }, + { + "id": "release-registry-self-test", + "description": "Exercise the release registry, descriptor, bundle, and negative parser cases.", + "commands": [existing_script_command("scripts/python_bin.sh", "scripts/check_test_wiring.py", "--self-test")], + "preflight_runnable": True, + }, + { + "id": "runner-self-tests", + "description": "Run lightweight CLI/schema tests for every Scanner/Heal evidence runner.", + "commands": [ + existing_script_command("scripts/run_scanner_heal_evidence_case.sh", "--self-test"), + existing_script_command("scripts/test_scanner_heal_authority_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_checkpoint_crash_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_status_outcome_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_scoped_ack_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_legacy_rollback_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_g14_multiset_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_g09_upgrade_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_scheduler_pressure_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_maintenance_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_w13_mrf_evidence.sh"), + existing_script_command("scripts/test_scanner_heal_w16_recovery_evidence.sh"), + ], + "preflight_runnable": True, + }, + ] + + +def concrete_case_steps(registry: dict[str, Any]) -> list[dict[str, Any]]: + cases = registry["cases"] + ordered_cases = [ + "background-target-restart", + "background-target-crash", + "background-target-restart-ec8-4", + "background-target-crash-ec8-4", + "ec84-target-drive-restart", + "background-target-restart-ec8-4-multi-set", + "background-target-crash-ec8-4-multi-pool", + ] + missing = [case for case in ordered_cases if case not in cases] + if missing: + raise ValueError(f"release registry is missing concrete cases: {', '.join(missing)}") + return [ + { + "id": f"case-{case_id}", + "description": cases[case_id]["scope"], + "covers": { + "gate": cases[case_id]["gate"], + "task": cases[case_id]["task"], + "lane": cases[case_id]["lane"], + "evidence": cases[case_id]["evidence"], + }, + "commands": [ + existing_script_command( + "scripts/run_scanner_heal_evidence_case.sh", + "--case", + case_id, + "--run-dir", + f"$RUN_ROOT/cases/{case_id}", + ) + ], + "expected_outputs": expected_outputs( + f"$RUN_ROOT/cases/{case_id}/run.json", + f"$RUN_ROOT/cases/{case_id}/execution.json", + f"$RUN_ROOT/cases/{case_id}/listing.json", + f"$RUN_ROOT/cases/{case_id}/junit.xml", + f"$RUN_ROOT/cases/{case_id}/{cases[case_id]['oracle']}", + f"$RUN_ROOT/cases/{case_id}/release-status.json", + ), + } + for case_id in ordered_cases + ] + + +def descriptor_steps() -> list[dict[str, Any]]: + return [ + { + "id": "authority-coverage", + "description": "Assemble the G01 authority descriptor from operator-collected root and quota authority artifacts.", + "covers": {"gates": ["G01"], "issues": ["2270"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_authority_evidence.py", + "--root-authority-json", + "$RUN_ROOT/raw/authority/root-authority.json", + "--quota-authority-json", + "$RUN_ROOT/raw/authority/quota-authority.json", + "--out-dir", + "$RUN_ROOT/descriptors/authority", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/descriptors/authority/release-bundle-authority.json"), + }, + { + "id": "checkpoint-and-restart", + "description": "Assemble G02/R-E bounded checkpoint and restart descriptors from measured diagnostic reports.", + "covers": {"gates": ["G02", "R-E"], "issues": ["2269"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_checkpoint_crash_evidence.py", + "--diagnostic-dir", + "$RUN_ROOT/raw/checkpoint", + "--out-dir", + "$RUN_ROOT/descriptors/checkpoint", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/descriptors/checkpoint/release-bundle-checkpoint-crash.json"), + }, + { + "id": "status-and-outcome", + "description": "Collect live G05/G06/R-D raw observations, then assemble their descriptor.", + "covers": {"gates": ["G05", "G06", "R-D"], "issues": ["2278"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_status_outcome_probe.py", + "--observations-json", + "$RUN_ROOT/raw/status-outcome/observations.json", + "--out-dir", + "$RUN_ROOT/raw/status-outcome", + ), + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_status_outcome_evidence.py", + "--status-outcome-json", + "$RUN_ROOT/raw/status-outcome/status-outcome.json", + "--status-compat-json", + "$RUN_ROOT/raw/status-outcome/status-compat.json", + "--disposition-json", + "$RUN_ROOT/raw/status-outcome/disposition.json", + "--out-dir", + "$RUN_ROOT/descriptors/status-outcome", + ), + ], + "expected_outputs": expected_outputs( + "$RUN_ROOT/raw/status-outcome/status-outcome.json", + "$RUN_ROOT/raw/status-outcome/status-compat.json", + "$RUN_ROOT/raw/status-outcome/disposition.json", + "$RUN_ROOT/descriptors/status-outcome/release-bundle-status-outcome.json", + ), + }, + { + "id": "ec8-4-multiset-descriptor", + "description": "Assemble G14 EC8+4 multi-set and multi-pool descriptors from the measured case directories.", + "covers": {"gates": ["G14"], "issues": ["2266", "2269"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_g14_multiset_evidence.py", + "--case-dir", + "multi-set=$RUN_ROOT/cases/background-target-restart-ec8-4-multi-set", + "--case-dir", + "multi-pool=$RUN_ROOT/cases/background-target-crash-ec8-4-multi-pool", + "--out-dir", + "$RUN_ROOT/descriptors/g14", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/descriptors/g14/release-bundle-g14.json"), + }, + { + "id": "w16-recovery-intent", + "description": "Run the G04/G12 recovery-intent and quota-authority lanes.", + "covers": {"gates": ["G04", "G12"], "issues": ["2279"]}, + "commands": [ + existing_script_command( + "scripts/run_scanner_heal_w16_recovery_evidence.sh", + "--run-dir", + "$RUN_ROOT/w16", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/w16/release-bundle-w16.json"), + }, + { + "id": "mrf-responsibility", + "description": "Run or package G07/G08/P4 durable MRF responsibility evidence.", + "covers": {"gates": ["G07", "G08", "P4"], "issues": ["2277", "2278"]}, + "commands": [ + existing_script_command( + "scripts/run_scanner_heal_w13_mrf_evidence.sh", + "--run-dir", + "$RUN_ROOT/w13", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/w13/release-bundle-w13.json"), + }, + { + "id": "mixed-version-rollback", + "description": "Run G09 mixed-version/rollback, then assemble G03 and R-L operator descriptors.", + "covers": {"gates": ["G03", "G09", "R-L"], "issues": ["2269", "2281"]}, + "commands": [ + existing_script_command( + "scripts/run_scanner_heal_g09_upgrade_evidence.sh", + "--run-dir", + "$RUN_ROOT/g09", + ), + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_scoped_ack_evidence.py", + "--proof-json", + "$RUN_ROOT/raw/scoped-ack/scoped-ack-proof.json", + "--out-dir", + "$RUN_ROOT/descriptors/scoped-ack", + ), + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_legacy_rollback_evidence.py", + "--proof-json", + "$RUN_ROOT/raw/legacy-rollback/legacy-rollback-proof.json", + "--out-dir", + "$RUN_ROOT/descriptors/legacy-rollback", + ), + ], + "expected_outputs": expected_outputs( + "$RUN_ROOT/g09/release-bundle-g09.json", + "$RUN_ROOT/descriptors/scoped-ack/release-bundle-scoped-ack.json", + "$RUN_ROOT/descriptors/legacy-rollback/release-bundle-legacy-rollback.json", + ), + }, + { + "id": "maintenance-producers", + "description": "Assemble G11/G13 producer coverage, quorum-minus-one, and remount descriptors.", + "covers": {"gates": ["G11", "G13"], "issues": ["2272", "2280"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_maintenance_evidence.py", + "--proof-json", + "$RUN_ROOT/raw/maintenance/maintenance-proof.json", + "--out-dir", + "$RUN_ROOT/descriptors/maintenance", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/descriptors/maintenance/release-bundle-maintenance.json"), + }, + ] + + +def performance_steps() -> list[dict[str, Any]]: + return [ + { + "id": "scanner-heal-abba", + "description": "Run the isolated EC8+4 Scanner/Heal ABBA matrix through the deployment adapter.", + "covers": {"gates": ["G10", "P1", "P2", "P3"], "issues": ["2266", "2273", "2274", "2275"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/scanner_abba.py", + "--manifest", + "$RUN_ROOT/abba/manifest.json", + "--adapter", + "$RUN_ROOT/abba/adapter.py", + "--out-dir", + "$RUN_ROOT/abba/out", + "--data-root", + "$RUN_ROOT/abba/data", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/abba/out/report.json"), + }, + { + "id": "scheduler-pressure", + "description": "Assemble scheduler-pressure, profile, RSS, throughput, and latency descriptors.", + "covers": {"gates": ["G10", "P1", "P2", "P3"], "issues": ["2266", "2274", "2275"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/run_scanner_heal_scheduler_pressure_evidence.py", + "--abba-dir", + "$RUN_ROOT/abba/out", + "--recovery-window-json", + "$RUN_ROOT/raw/scheduler/recovery-window.json", + "--profile-artifact", + "allocation-profile=$RUN_ROOT/profile/allocation-profile.json", + "--profile-artifact", + "flamegraph=$RUN_ROOT/profile/flamegraph.svg", + "--profile-artifact", + "rss-samples=$RUN_ROOT/profile/rss-samples.json", + "--profile-artifact", + "save-frequency=$RUN_ROOT/profile/save-frequency.json", + "--out-dir", + "$RUN_ROOT/descriptors/scheduler-pressure", + ) + ], + "expected_outputs": expected_outputs("$RUN_ROOT/descriptors/scheduler-pressure/release-bundle-scheduler-pressure.json"), + }, + ] + + +def bundle_steps() -> list[dict[str, Any]]: + return [ + { + "id": "assemble-release-bundle", + "description": "Assemble every measured lane descriptor into the final release bundle.", + "covers": {"gates": sorted(REQUIRED_GATES), "issues": ["2240", "2428"]}, + "commands": [ + existing_script_command( + "scripts/python_bin.sh", + "scripts/check_test_wiring.py", + "--assemble-scanner-heal-release-bundle", + "$RUN_ROOT/descriptors/authority/release-bundle-authority.json", + "$RUN_ROOT/descriptors/checkpoint/release-bundle-checkpoint-crash.json", + "$RUN_ROOT/descriptors/status-outcome/release-bundle-status-outcome.json", + "$RUN_ROOT/w16/release-bundle-w16.json", + "$RUN_ROOT/w13/release-bundle-w13.json", + "$RUN_ROOT/g09/release-bundle-g09.json", + "$RUN_ROOT/descriptors/scoped-ack/release-bundle-scoped-ack.json", + "$RUN_ROOT/descriptors/legacy-rollback/release-bundle-legacy-rollback.json", + "$RUN_ROOT/descriptors/maintenance/release-bundle-maintenance.json", + "$RUN_ROOT/descriptors/g14/release-bundle-g14.json", + "$RUN_ROOT/descriptors/scheduler-pressure/release-bundle-scheduler-pressure.json", + "$RUN_ROOT/release-bundle", + ), + existing_script_command( + "scripts/python_bin.sh", + "scripts/check_test_wiring.py", + "--check-scanner-heal-release-bundle", + "$RUN_ROOT/release-bundle/release-evidence.json", + ), + ], + "expected_outputs": expected_outputs("$RUN_ROOT/release-bundle/release-evidence.json"), + } + ] + + +def build_plan(registry: dict[str, Any], revision: str, phases: set[str]) -> dict[str, Any]: + stage_defs = [ + ("preflight", "Preflight", preflight_steps()), + ("functional", "Functional And Durable Evidence", concrete_case_steps(registry) + descriptor_steps()), + ("performance", "ABBA And Profile Evidence", performance_steps()), + ("bundle", "Release Bundle Assembly", bundle_steps()), + ] + stages = [] + for key, title, steps in stage_defs: + if "all" not in phases and key not in phases: + continue + stages.append({"id": key, "title": title, "steps": steps}) + return { + "schema": 1, + "kind": "scanner-heal-linux-evidence-plan", + "evidence_type": "plan_only", + "source_revision": revision, + "registry": str(REGISTRY.relative_to(ROOT)), + "run_root_env": "RUN_ROOT", + "requirements": { + "base_branch": "release", + "platform": "Linux", + "tracked_source_clean": True, + "measured_evidence_required": True, + "synthetic_evidence_rejected": True, + "stop_on_product_failure": True, + }, + "stages": stages, + } + + +def write_plan(out_dir: Path, plan: dict[str, Any]) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / "scanner-heal-linux-evidence-plan.json" + path.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n") + return path + + +def text_plan(plan: dict[str, Any]) -> str: + lines = [ + f"kind={plan['kind']}", + f"source_revision={plan['source_revision']}", + f"registry={plan['registry']}", + "evidence_type=plan_only", + ] + for stage in plan["stages"]: + lines.append(f"stage={stage['id']} steps={len(stage['steps'])}") + for step in stage["steps"]: + lines.append(f" step={step['id']}") + for entry in step["commands"]: + lines.append(" command=" + " ".join(entry["command"])) + return "\n".join(lines) + + +def iter_preflight_commands(plan: dict[str, Any]) -> list[list[str]]: + commands: list[list[str]] = [] + for stage in plan["stages"]: + if stage["id"] != "preflight": + continue + for step in stage["steps"]: + if not step.get("preflight_runnable"): + continue + commands.extend(entry["command"] for entry in step["commands"]) + return commands + + +def render_run_root_path(value: str, run_root: Path) -> Path: + rendered = value.replace("$RUN_ROOT", str(run_root)) + return Path(rendered) + + +def check_expected_output(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"path": str(path), "status": "missing"} + if not path.is_file(): + return {"path": str(path), "status": "invalid", "error": "expected a file"} + size = path.stat().st_size + if size <= 0: + return {"path": str(path), "status": "empty", "bytes": size} + return {"path": str(path), "status": "present", "bytes": size} + + +def covered_gates(step: dict[str, Any]) -> list[str]: + covers = step.get("covers") + if not isinstance(covers, dict): + return [] + gate = covers.get("gate") + gates = covers.get("gates") + if isinstance(gate, str): + return [gate] + if isinstance(gates, list): + return [item for item in gates if isinstance(item, str)] + return [] + + +def build_status(plan: dict[str, Any], run_root: Path) -> dict[str, Any]: + run_root = run_root.resolve() + stage_statuses = [] + totals = {"present": 0, "missing": 0, "empty": 0, "invalid": 0, "expected": 0} + pending_gates: set[str] = set() + next_step: dict[str, Any] | None = None + for stage in plan["stages"]: + step_statuses = [] + for step in stage["steps"]: + outputs = [ + check_expected_output(render_run_root_path(path, run_root)) + for path in step.get("expected_outputs", []) + ] + counts = {"present": 0, "missing": 0, "empty": 0, "invalid": 0} + for output in outputs: + counts[output["status"]] += 1 + for key in counts: + totals[key] += counts[key] + totals["expected"] += len(outputs) + if not outputs: + status = "not_tracked" + elif counts["invalid"] or counts["empty"]: + status = "invalid" + elif counts["missing"]: + status = "pending" if counts["present"] == 0 else "partial" + else: + status = "complete" + if status in {"pending", "partial", "invalid"}: + pending_gates.update(covered_gates(step)) + if next_step is None: + next_step = { + "stage": stage["id"], + "step": step["id"], + "status": status, + "commands": step["commands"], + } + step_statuses.append({ + "id": step["id"], + "status": status, + "covers": step.get("covers", {}), + "outputs": outputs, + }) + tracked = [step for step in step_statuses if step["status"] != "not_tracked"] + if not tracked: + stage_state = "not_tracked" + elif all(step["status"] == "complete" for step in tracked): + stage_state = "complete" + elif any(step["status"] == "invalid" for step in tracked): + stage_state = "invalid" + elif any(step["status"] in {"complete", "partial"} for step in tracked): + stage_state = "partial" + else: + stage_state = "pending" + stage_statuses.append({"id": stage["id"], "status": stage_state, "steps": step_statuses}) + if totals["invalid"] or totals["empty"]: + decision = "invalid" + elif totals["missing"]: + decision = "blocked" + else: + decision = "complete" + return { + "schema": 1, + "kind": "scanner-heal-linux-evidence-status", + "decision": decision, + "release_approved": False, + "source_revision": plan["source_revision"], + "run_root": str(run_root), + "artifact_totals": totals, + "pending_gates": sorted(pending_gates), + "next_step": next_step, + "stages": stage_statuses, + } + + +def run_preflight(plan: dict[str, Any]) -> int: + commands = iter_preflight_commands(plan) + if not commands: + raise ValueError("--run-preflight requires the preflight stage") + failures = [] + for args in commands: + result = subprocess.run(args, cwd=ROOT) + if result.returncode != 0: + failures.append({"command": args, "exit_code": result.returncode}) + break + if failures: + print(json.dumps({"status": "failed", "failures": failures}, indent=2), file=sys.stderr) + return 1 + print(json.dumps({"status": "passed", "commands": len(commands)}, indent=2)) + return 0 + + +def self_test() -> None: + registry = load_registry() + validate_registry(registry) + revision = git_output("rev-parse", "HEAD") + plan = build_plan(registry, revision, {"all"}) + assert plan["evidence_type"] == "plan_only" + stages = {stage["id"]: stage for stage in plan["stages"]} + assert set(stages) == {"preflight", "functional", "performance", "bundle"} + commands = [ + " ".join(entry["command"]) + for stage in plan["stages"] + for step in stage["steps"] + for entry in step["commands"] + ] + assert not any("--case release" in item for item in commands) + assert any("run_scanner_heal_g09_upgrade_evidence.sh" in item for item in commands) + assert any("scanner_abba.py" in item for item in commands) + assert any("--check-scanner-heal-release-bundle" in item for item in commands) + preflight = iter_preflight_commands(plan) + assert preflight + assert all(args[0].startswith("scripts/") for args in preflight) + out_dir = Path(os.environ.get("TMPDIR", "/tmp")) / "rustfs-scanner-heal-linux-plan-self-test" + path = write_plan(out_dir, plan) + loaded = json.loads(path.read_text()) + assert loaded["source_revision"] == revision + status = build_status(plan, out_dir / "empty-run") + assert status["decision"] == "blocked" + assert status["release_approved"] is False + assert status["pending_gates"] + complete_root = out_dir / "complete-run" + for stage in plan["stages"]: + for step in stage["steps"]: + for output in step.get("expected_outputs", []): + path = render_run_root_path(output, complete_root) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n") + complete = build_status(plan, complete_root) + assert complete["decision"] == "complete" + assert complete["release_approved"] is False + print("PASS: scanner/heal Linux evidence plan self-test") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--phase", action="append", choices=("all", "preflight", "functional", "performance", "bundle"), default=[]) + parser.add_argument("--source-revision") + parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUTPUT_ROOT) + parser.add_argument("--write-plan", action="store_true") + parser.add_argument("--format", choices=("text", "json"), default="text") + parser.add_argument("--run-preflight", action="store_true") + parser.add_argument("--status-root", type=Path) + parser.add_argument("--self-test", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if args.self_test: + self_test() + return 0 + try: + registry = load_registry() + validate_registry(registry) + phases = set(args.phase or ["all"]) + if "all" in phases and len(phases) > 1: + raise ValueError("--phase all cannot be combined with another phase") + if args.status_root is not None and (args.write_plan or args.run_preflight): + raise ValueError("--status-root cannot be combined with --write-plan or --run-preflight") + plan = build_plan(registry, source_revision(args.source_revision), phases) + if args.status_root is not None: + status = build_status(plan, args.status_root) + print(json.dumps(status, indent=2, sort_keys=True)) + return 0 if status["decision"] == "complete" else 3 + if args.write_plan: + path = write_plan(args.out_dir.resolve(), plan) + print(path) + elif args.run_preflight: + pass + elif args.format == "json": + print(json.dumps(plan, indent=2, sort_keys=True)) + else: + print(text_plan(plan)) + if args.run_preflight: + return run_preflight(plan) + return 0 + except (AssertionError, ValueError, OSError, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_maintenance_evidence.py b/scripts/run_scanner_heal_maintenance_evidence.py new file mode 100755 index 000000000..1fc7a5b9c --- /dev/null +++ b/scripts/run_scanner_heal_maintenance_evidence.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal G11/G13 maintenance release evidence. + +The producer consumes operator-collected measured JSON. It only packages and +checks the evidence fields; it does not run the distributed workload or approve +the full release gate by itself. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import check_test_wiring as wiring + +ROOT = Path(__file__).resolve().parents[1] +G11_FIELDS = ( + "maintenance_producer_matrix", + "complete_producer_inventory", + "segment_activation_preflight", +) +G13_FIELDS = ( + "quorum_minus_one_matrix", + "unknown_disk_remount_matrix", + "object_lock_dry_run_grace_evidence", +) + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def timestamp(value: Any, name: str) -> str: + wiring.require(isinstance(value, str) and value.strip(), f"missing {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + wiring.require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def measured_proof(path: Path, source_revision: str) -> dict[str, Any]: + proof = wiring.read_json(path) + wiring.require(proof.get("schema") == 1, "proof schema must be 1") + wiring.require(proof.get("evidence_type") == "measured", "proof must be measured") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(proof.get(marker) is not True, f"proof is {marker}") + wiring.require(proof.get("source_revision") == source_revision, "proof source revision mismatch") + timestamp(proof.get("started_at"), "proof.started_at") + timestamp(proof.get("finished_at"), "proof.finished_at") + wiring.evidence_string(proof.get("run_id"), "proof.run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + wiring.evidence_string( + proof.get("measurement_window_id"), + "proof.measurement_window_id", + r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", + ) + wiring.require(proof["measurement_window_id"] != proof["run_id"], "proof must separate run/window identities") + return proof + + +def field_from_proof(proof: dict[str, Any], field: str) -> dict[str, Any]: + value = proof.get(field) + wiring.require(isinstance(value, dict), f"proof missing {field}") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(value.get(marker) is not True, f"{field} is {marker}") + if "evidence_type" in value: + wiring.require(value["evidence_type"] == "measured", f"{field} must be measured") + return dict(value) + + +def write_field(out_dir: Path, gate: str, field: str, common: dict[str, Any], field_evidence: dict[str, Any]) -> dict[str, Any]: + evidence = { + **common, + **field_evidence, + "evidence_type": "measured", + "summary": field_evidence.get("summary") or f"Measured Scanner/Heal {gate}.{field} evidence.", + } + artifact = out_dir / "artifacts" / f"{gate}-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "gate": gate, + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + wiring.write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = wiring.digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + wiring.require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + proof = measured_proof(args.proof_json.resolve(), source_revision) + out_dir.mkdir(parents=True) + common = { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "started_at": timestamp(proof["started_at"], "proof.started_at"), + "finished_at": timestamp(proof["finished_at"], "proof.finished_at"), + "command": [ + "scripts/run_scanner_heal_maintenance_evidence.py", + "--proof-json", + "", + ], + } + gates = { + "G11": { + "status": "pass", + "lane": "maintenance-producers", + "evidence_type": "measured", + "evidence_fields": { + field: write_field(out_dir, "G11", field, common, field_from_proof(proof, field)) + for field in G11_FIELDS + }, + }, + "G13": { + "status": "pass", + "lane": "maintenance-producers", + "evidence_type": "measured", + "evidence_fields": { + field: write_field(out_dir, "G13", field, common, field_from_proof(proof, field)) + for field in G13_FIELDS + }, + }, + } + descriptor = out_dir / "release-bundle-maintenance.json" + wiring.write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + for gate in ("G11", "G13"): + check = subprocess.run([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + gate, + ], cwd=ROOT, capture_output=True, text=True) + if check.returncode != 0: + details = "\n".join(part for part in (check.stdout.strip(), check.stderr.strip()) if part) + raise ValueError(details or f"{gate} release bundle gate check failed") + return descriptor + + +def write_self_test_proof(path: Path, source_revision: str) -> None: + now = datetime.now(timezone.utc).replace(microsecond=0) + proof = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": f"maintenance-{source_revision[:12]}", + "measurement_window_id": f"maintenance-window-{source_revision[:12]}", + "started_at": now.isoformat().replace("+00:00", "Z"), + "finished_at": now.isoformat().replace("+00:00", "Z"), + "maintenance_producer_matrix": { + "producer_identities": list(wiring.SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES), + "producer_families": list(wiring.SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES), + "matrix_cases": list(wiring.SCANNER_HEAL_RELEASE_G11_REQUIRED_CASES["maintenance_producer_matrix"]), + "durable_identity_observed": True, + "durable_journal_replay_observed": True, + "durable_journal_hydration_bound": True, + "generation_window_bound": True, + "restart_gap_absent": True, + "overflow_absent": True, + "ack_clear_generation_observed": True, + "invalid_journal_inputs_rejected": list(wiring.SCANNER_HEAL_DURABLE_JOURNAL_FAIL_CLOSED_CASES), + }, + "complete_producer_inventory": { + "required_producer_identities": list(wiring.SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES), + "observed_producer_identities": list(wiring.SCANNER_HEAL_REQUIRED_PRODUCER_IDENTITIES), + "required_producer_families": list(wiring.SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES), + "observed_producer_families": list(wiring.SCANNER_HEAL_REQUIRED_PRODUCER_FAMILIES), + "missing_producer_identities": [], + "unknown_producer_excluded": True, + }, + "segment_activation_preflight": { + "production_activation": True, + "scanner_segment_reuse_activated": True, + "proof_inputs": list(wiring.SCANNER_HEAL_SEGMENT_ACTIVATION_PROOF_INPUTS), + "fail_closed_checks": list(wiring.SCANNER_HEAL_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS), + }, + "quorum_minus_one_matrix": { + "quorum_cases": list(wiring.SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES["quorum_minus_one_matrix"]), + "no_success_at_quorum_minus_one": True, + "exact_quorum_restored": True, + }, + "unknown_disk_remount_matrix": { + "remount_cases": list(wiring.SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES["unknown_disk_remount_matrix"]), + "unknown_disks_excluded": True, + "remounted_disks_revalidated": True, + "stale_incarnation_rejected": True, + }, + "object_lock_dry_run_grace_evidence": { + "grace_cases": list(wiring.SCANNER_HEAL_RELEASE_G13_REQUIRED_CASES["object_lock_dry_run_grace_evidence"]), + "object_lock_denials_preserved": True, + "dry_run_mutation_count": 0, + "grace_outcomes_retained": True, + }, + } + wiring.write_json(path, proof) + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "maintenance-proof.json" + write_self_test_proof(proof, source_revision) + descriptor = build_descriptor(parse_args([ + "--proof-json", str(proof), + "--out-dir", str(root / "out"), + ])) + wiring.require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "maintenance-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["evidence_type"] = "synthetic" + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("measured" in str(err), "wrong self-test failure for synthetic proof") + else: + raise ValueError("self-test accepted synthetic proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "maintenance-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["segment_activation_preflight"]["scanner_segment_reuse_activated"] = False + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("runtime activation gate is enabled" in str(err), "wrong self-test failure for inactive segment reuse") + else: + raise ValueError("self-test accepted inactive segment reuse proof") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proof-json", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.proof_json is None: + parser.error("--proof-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Maintenance release descriptor verified: {descriptor}") + return 0 + except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_mrf_evidence.py b/scripts/run_scanner_heal_mrf_evidence.py new file mode 100755 index 000000000..22a1836e6 --- /dev/null +++ b/scripts/run_scanner_heal_mrf_evidence.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal G07/G08/P4 MRF release evidence. + +The producer consumes raw W13 MRF JSON artifacts emitted by the measured Rust +test and packages them into the common release-bundle descriptor shape. It is +usable both by the W13 shell runner and by operators who need to re-check an +already collected MRF evidence directory. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import check_test_wiring as wiring + +ROOT = Path(__file__).resolve().parents[1] +FIELD_ARTIFACTS = { + "G07": { + "mrf_responsibility_oracle": "g07-mrf-responsibility/G07-mrf_responsibility_oracle.json", + "commit_boundary_crash_matrix": "g07-mrf-responsibility/G07-commit_boundary_crash_matrix.json", + }, + "G08": { + "mrf_capacity_evidence": "g08-mrf-capacity/G08-mrf_capacity_evidence.json", + "disk_full_matrix": "g08-mrf-capacity/G08-disk_full_matrix.json", + "replica_loss_matrix": "g08-mrf-capacity/G08-replica_loss_matrix.json", + }, + "P4": { + "mrf_scale_measurement": "p4-mrf-soak/P4-mrf_scale_measurement.json", + "mrf_replay_cost_measurement": "p4-mrf-soak/P4-mrf_replay_cost_measurement.json", + "retained_responsibility_evidence": "p4-mrf-soak/P4-retained_responsibility_evidence.json", + "mrf_cleanup_gc_soak_evidence": "p4-mrf-soak/P4-mrf_cleanup_gc_soak_evidence.json", + }, +} +SELECTION_GATES = { + "all": ("G07", "G08", "P4"), + "g07": ("G07",), + "g08": ("G08",), + "p4": ("P4",), +} + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def source_revision_for(run_dir: Path, override: str | None) -> str: + if override: + return override + stamp = run_dir / "source-revision.txt" + if stamp.is_file(): + value = stamp.read_text().strip() + wiring.require(value, "source-revision.txt is empty") + return value + return git_head() + + +def timestamp(value: Any, name: str) -> str: + wiring.require(isinstance(value, str) and value.strip(), f"missing {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + wiring.require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def selected_artifacts(selection: str) -> dict[str, dict[str, Path]]: + return { + gate: {field: Path(relative) for field, relative in FIELD_ARTIFACTS[gate].items()} + for gate in SELECTION_GATES[selection] + } + + +def descriptor_path_for(run_dir: Path, output: Path | None) -> Path: + if output is not None: + return output.resolve() + return run_dir / "release-bundle-w13.json" + + +def relative_to_descriptor(path: Path, descriptor: Path) -> str: + return path.resolve(strict=True).relative_to(descriptor.parent.resolve()).as_posix() + + +def measured_payload(path: Path, source_revision: str, gate: str, field: str) -> dict[str, Any]: + payload = wiring.read_json(path.resolve()) + wiring.require(isinstance(payload, dict), f"{gate}.{field} artifact must be a JSON object") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(payload.get(marker) is not True, f"{gate}.{field} artifact is {marker}") + wiring.require(payload.get("schema") == 1, f"{gate}.{field} artifact schema must be 1") + wiring.require(payload.get("evidence_type") == "measured", f"{gate}.{field} artifact must be measured") + wiring.require(payload.get("source_revision") == source_revision, f"{gate}.{field} source revision mismatch") + wiring.require(payload.get("gate") == gate, f"{gate}.{field} artifact gate mismatch") + wiring.require(payload.get("field") == field, f"{gate}.{field} artifact field mismatch") + for key in ("run_id", "measurement_window_id"): + value = payload.get(key) + wiring.require(isinstance(value, str) and value.strip(), f"{gate}.{field} missing {key}") + expected_kind = wiring.SCANNER_HEAL_RELEASE_MRF_ARTIFACT_KINDS[(gate, field)] + wiring.require(payload.get("artifact_kind") == expected_kind, f"{gate}.{field} artifact kind mismatch") + timestamp(payload.get("started_at"), f"{gate}.{field}.started_at") + timestamp(payload.get("finished_at"), f"{gate}.{field}.finished_at") + command = payload.get("command") + wiring.require( + isinstance(command, list) and command and all(isinstance(item, str) and item.strip() for item in command), + f"{gate}.{field} artifact missing command provenance", + ) + summary = payload.get("summary") + wiring.require(isinstance(summary, str) and summary.strip(), f"{gate}.{field} artifact missing summary") + if gate == "G08" and field == "disk_full_matrix": + for observed in ( + "journal_write_enospc_observed", + "committed_checkpoint_enospc_observed", + "cleanup_delete_on_full_filesystem_observed", + ): + wiring.release_bundle_bool_true(payload.get(observed), f"{gate}.{field}.{observed}") + wiring.evidence_integer(payload.get("enospc_filler_bytes"), f"{gate}.{field}.enospc_filler_bytes", 1, 2**63 - 1) + return payload + + +def validate_mrf_field(gate: str, field: str, evidence: dict[str, Any]) -> None: + if gate == "G07": + case_field = { + "mrf_responsibility_oracle": "mrf_responsibility_cases", + "commit_boundary_crash_matrix": "commit_crash_cases", + }[field] + wiring.release_bundle_exact_strings( + evidence.get(case_field), + wiring.SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES[field], + f"{gate}.{field}.{case_field}", + ) + wiring.evidence_integer(evidence.get("replayed_records"), f"{gate}.{field}.replayed_records", 1, 2**63 - 1) + wiring.release_bundle_bool_true( + evidence.get("responsibility_anchor_retained"), + f"{gate}.{field}.responsibility_anchor_retained", + ) + wiring.release_bundle_bool_true( + evidence.get("successor_snapshot_published"), + f"{gate}.{field}.successor_snapshot_published", + ) + if gate == "G08": + case_field = { + "mrf_capacity_evidence": "capacity_cases", + "disk_full_matrix": "disk_full_cases", + "replica_loss_matrix": "replica_loss_cases", + }[field] + wiring.release_bundle_exact_strings( + evidence.get(case_field), + wiring.SCANNER_HEAL_RELEASE_G08_REQUIRED_CASES[field], + f"{gate}.{field}.{case_field}", + ) + if gate == "P4": + duration = wiring.evidence_integer(evidence.get("duration_seconds"), f"{gate}.{field}.duration_seconds", 1, 86400) + wiring.require(duration >= 900, f"{gate}.{field} requires at least 900 seconds") + if field != "mrf_scale_measurement": + wiring.evidence_integer(evidence.get("replayed_records"), f"{gate}.{field}.replayed_records", 1, 2**63 - 1) + wiring.release_bundle_bool_true( + evidence.get("responsibility_anchor_retained"), + f"{gate}.{field}.responsibility_anchor_retained", + ) + wiring.release_bundle_bool_true( + evidence.get("successor_snapshot_published"), + f"{gate}.{field}.successor_snapshot_published", + ) + if field == "retained_responsibility_evidence": + wiring.release_bundle_exact_strings( + evidence.get("retained_responsibility_cases"), + wiring.SCANNER_HEAL_RELEASE_P4_RETAINED_RESPONSIBILITY_CASES, + f"{gate}.{field}.retained_responsibility_cases", + ) + retention_window = wiring.evidence_integer( + evidence.get("retention_window_seconds"), + f"{gate}.{field}.retention_window_seconds", + 7200, + 86400, + ) + wiring.require(duration >= retention_window, f"{gate}.{field} duration must cover retention window") + wiring.release_bundle_bool_true(evidence.get("idle_cleanup_observed"), f"{gate}.{field}.idle_cleanup_observed") + wiring.release_bundle_bool_true( + evidence.get("verified_proof_discharge_observed"), + f"{gate}.{field}.verified_proof_discharge_observed", + ) + if field == "mrf_cleanup_gc_soak_evidence": + wiring.require(duration >= 7200, f"{gate}.{field} requires at least two hours") + wiring.release_bundle_exact_strings( + evidence.get("cleanup_gc_cases"), + wiring.SCANNER_HEAL_RELEASE_MRF_CLEANUP_GC_SOAK_CASES, + f"{gate}.{field}.cleanup_gc_cases", + ) + wiring.release_bundle_bool_true( + evidence.get("verified_idle_gc_observed"), + f"{gate}.{field}.verified_idle_gc_observed", + ) + wiring.require(evidence.get("pending_responsibilities_after_gc") == 0, + f"{gate}.{field} requires zero pending responsibilities after GC") + wiring.require(evidence.get("stale_journals_after_gc") == 0, + f"{gate}.{field} requires zero stale journals after GC") + + +def field_evidence(run_dir: Path, descriptor: Path, source_revision: str, gate: str, field: str, relative: Path) -> dict[str, Any]: + artifact = run_dir / relative + payload = measured_payload(artifact, source_revision, gate, field) + evidence = { + "artifact": relative_to_descriptor(artifact, descriptor), + "sha256": wiring.digest(artifact), + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": payload["run_id"], + "measurement_window_id": payload["measurement_window_id"], + "started_at": payload["started_at"], + "finished_at": payload["finished_at"], + "command": payload["command"], + "artifact_format": "json", + "summary": payload["summary"], + } + for mirror in wiring.release_bundle_json_artifact_mirrored_fields(gate, field): + wiring.require(mirror in payload, f"{gate}.{field} artifact missing {mirror}") + evidence[mirror] = payload[mirror] + if gate == "P4": + evidence["duration_seconds"] = payload["duration_seconds"] + validate_mrf_field(gate, field, evidence) + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + run_dir = args.run_dir.resolve() + wiring.require(run_dir.is_dir(), "run directory is missing") + descriptor = descriptor_path_for(run_dir, args.out_file) + wiring.require(descriptor.parent == run_dir or descriptor.parent.is_relative_to(run_dir), + "descriptor must be written under the run directory") + source_revision = source_revision_for(run_dir, args.source_revision) + registry = wiring.read_json(ROOT / ".config/scanner-heal-required-tests.json") + requirements = {item["gate"]: item for item in registry["release_requirements"]} + gates: dict[str, Any] = {} + for gate, artifacts in selected_artifacts(args.test).items(): + gates[gate] = { + "status": "pass", + "lane": requirements[gate]["lane"], + "evidence_type": "measured", + "evidence_fields": { + field: field_evidence(run_dir, descriptor, source_revision, gate, field, relative) + for field, relative in artifacts.items() + }, + } + wiring.write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + for gate in gates: + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + gate, + ], cwd=ROOT) + return descriptor + + +def base_payload(source_revision: str, gate: str, field: str, duration_seconds: int = 7200) -> dict[str, Any]: + started = datetime.now(timezone.utc).replace(microsecond=0) + finished = started + timedelta(seconds=duration_seconds) + return { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": f"w13-mrf-{gate.lower()}-{source_revision[:12]}", + "measurement_window_id": f"w13-mrf-window-{gate.lower()}-{source_revision[:12]}", + "started_at": started.isoformat().replace("+00:00", "Z"), + "finished_at": finished.isoformat().replace("+00:00", "Z"), + "command": ["scripts/run_scanner_heal_w13_mrf_evidence.sh", "--test", gate.lower()], + "summary": f"Measured Scanner/Heal {gate}.{field} MRF evidence.", + "gate": gate, + "field": field, + "artifact_kind": wiring.SCANNER_HEAL_RELEASE_MRF_ARTIFACT_KINDS[(gate, field)], + } + + +def write_self_test_artifacts(run_dir: Path, source_revision: str) -> None: + for gate, artifacts in selected_artifacts("all").items(): + for field, relative in artifacts.items(): + payload = base_payload(source_revision, gate, field) + if gate == "G07": + payload["crash_points"] = ["before-commit", "after-commit"] + payload["replayed_records"] = 2 + payload["responsibility_anchor_retained"] = True + payload["successor_snapshot_published"] = True + if field == "mrf_responsibility_oracle": + payload["mrf_responsibility_cases"] = list(wiring.SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES[field]) + else: + payload["commit_crash_cases"] = list(wiring.SCANNER_HEAL_RELEASE_G07_REQUIRED_CASES[field]) + elif gate == "G08": + case_field = { + "mrf_capacity_evidence": "capacity_cases", + "disk_full_matrix": "disk_full_cases", + "replica_loss_matrix": "replica_loss_cases", + }[field] + payload[case_field] = list(wiring.SCANNER_HEAL_RELEASE_G08_REQUIRED_CASES[field]) + if field == "disk_full_matrix": + payload.update({ + "journal_write_enospc_observed": True, + "committed_checkpoint_enospc_observed": True, + "cleanup_delete_on_full_filesystem_observed": True, + "enospc_filler_bytes": 1024, + }) + elif gate == "P4": + payload["duration_seconds"] = 7200 + if field != "mrf_scale_measurement": + payload["replayed_records"] = 2 + payload["responsibility_anchor_retained"] = True + payload["successor_snapshot_published"] = True + if field == "retained_responsibility_evidence": + payload.update({ + "retained_responsibility_cases": list(wiring.SCANNER_HEAL_RELEASE_P4_RETAINED_RESPONSIBILITY_CASES), + "retention_window_seconds": 7200, + "idle_cleanup_observed": True, + "verified_proof_discharge_observed": True, + }) + if field == "mrf_cleanup_gc_soak_evidence": + payload.update({ + "cleanup_gc_cases": list(wiring.SCANNER_HEAL_RELEASE_MRF_CLEANUP_GC_SOAK_CASES), + "verified_idle_gc_observed": True, + "pending_responsibilities_after_gc": 0, + "stale_journals_after_gc": 0, + }) + path = run_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + wiring.write_json(path, payload) + + +def expect_failure(args: list[str], needle: str) -> None: + try: + build_descriptor(parse_args(args)) + except (ValueError, subprocess.CalledProcessError) as err: + wiring.require(needle in str(err), f"wrong self-test failure: {err}") + else: + raise ValueError("self-test accepted invalid MRF evidence") + + +def run_self_test() -> None: + import tempfile + + source_revision = git_head() + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + run_dir.mkdir() + (run_dir / "source-revision.txt").write_text(source_revision + "\n") + write_self_test_artifacts(run_dir, source_revision) + descriptor = build_descriptor(parse_args(["--run-dir", str(run_dir)])) + wiring.require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + run_dir.mkdir() + write_self_test_artifacts(run_dir, source_revision) + path = run_dir / FIELD_ARTIFACTS["G07"]["mrf_responsibility_oracle"] + payload = wiring.read_json(path) + payload["mrf_responsibility_cases"] = payload["mrf_responsibility_cases"][:-1] + wiring.write_json(path, payload) + expect_failure(["--run-dir", str(run_dir), "--source-revision", source_revision, "--test", "g07"], "missing cases") + + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + run_dir.mkdir() + write_self_test_artifacts(run_dir, source_revision) + path = run_dir / FIELD_ARTIFACTS["G08"]["disk_full_matrix"] + payload = wiring.read_json(path) + payload["journal_write_enospc_observed"] = False + wiring.write_json(path, payload) + expect_failure(["--run-dir", str(run_dir), "--source-revision", source_revision, "--test", "g08"], "journal_write_enospc_observed") + + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + run_dir.mkdir() + write_self_test_artifacts(run_dir, source_revision) + path = run_dir / FIELD_ARTIFACTS["P4"]["mrf_cleanup_gc_soak_evidence"] + payload = wiring.read_json(path) + payload["duration_seconds"] = 900 + payload["finished_at"] = payload["started_at"] + wiring.write_json(path, payload) + expect_failure(["--run-dir", str(run_dir), "--source-revision", source_revision, "--test", "p4"], "two hours") + + with tempfile.TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "run" + run_dir.mkdir() + write_self_test_artifacts(run_dir, source_revision) + path = run_dir / FIELD_ARTIFACTS["G08"]["replica_loss_matrix"] + payload = wiring.read_json(path) + payload["synthetic"] = True + wiring.write_json(path, payload) + expect_failure(["--run-dir", str(run_dir), "--source-revision", source_revision, "--test", "g08"], "synthetic") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path) + parser.add_argument("--out-file", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--test", choices=tuple(SELECTION_GATES), default="all") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test and args.run_dir is None: + parser.error("--run-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Scanner/Heal MRF release descriptor verified: {descriptor}") + return 0 + except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_scheduler_pressure_evidence.py b/scripts/run_scanner_heal_scheduler_pressure_evidence.py new file mode 100755 index 000000000..0fff0136c --- /dev/null +++ b/scripts/run_scanner_heal_scheduler_pressure_evidence.py @@ -0,0 +1,691 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal scheduler-pressure release evidence. + +This producer consumes a completed measured Scanner/Heal ABBA run plus measured +profile and recovery-window artifacts. It only packages existing measurements; +it never turns synthetic harness output into release evidence. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import math +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +from scanner_abba import ( + P2_WORK_MULTIPLE_LIMIT, + RELEASE_PROFILE_ARTIFACTS, + RELEASE_SCHEDULER_BOUNDS, + SCENARIOS, + digest, + read_json, + require, + sha, + write_json, +) + +ROOT = Path(__file__).resolve().parents[1] +G10_FIELDS = ("scheduler_bound_evidence", "pressure_recovery_evidence") +P1_FIELDS = ("cold_walk_share_measurement", "foreground_latency_throughput_measurement", "profile_evidence") +P2_FIELDS = ("post_stop_convergence_measurement", "cold_segment_reuse_measurement") +P3_FIELDS = ("two_hour_pressure_measurement", "heal_capacity_measurement", "recovery_window_measurement") +PRESSURE_METRICS = ( + "foreground_p95_ms", + "foreground_p99_ms", + "throughput_ops", + "error_rate", + "heal_lock_wait_p99_ms", + "attempt_cost_samples", +) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def finite_number(value: Any, name: str, minimum: float = 0.0) -> float: + require(type(value) in (int, float) and math.isfinite(value) and value >= minimum, f"invalid {name}") + return float(value) + + +def positive_int(value: Any, name: str, minimum: int = 1) -> int: + require(type(value) is int and value >= minimum, f"invalid {name}") + return value + + +def positive_int_from_sources(cli_value: int | None, manifest_values: dict[str, Any], key: str) -> int: + if cli_value is not None: + return positive_int(cli_value, key) + return positive_int(manifest_values.get(key), key) + + +def parse_artifact_arg(value: str) -> tuple[str, Path, str]: + try: + kind, raw_path = value.split("=", 1) + except ValueError as err: + raise argparse.ArgumentTypeError("profile artifact must be KIND=PATH") from err + if kind not in RELEASE_PROFILE_ARTIFACTS: + raise argparse.ArgumentTypeError(f"unknown profile artifact kind: {kind}") + path = Path(raw_path).expanduser().resolve() + if not path.is_file() or path.stat().st_size == 0: + raise argparse.ArgumentTypeError(f"profile artifact is missing or empty: {raw_path}") + suffix = path.suffix.lower().lstrip(".") + artifact_format = f"profile-{suffix}" if suffix in {"json", "ndjson"} else (suffix if suffix else "binary") + return kind, path, artifact_format + + +def load_measured_abba(abba_dir: Path, source_revision: str) -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]: + manifest = read_json(abba_dir / "manifest.json") + report = read_json(abba_dir / "report.json") + require(manifest.get("evidence") == "measured", "ABBA manifest must be measured") + require(report.get("evidence") == "measured", "ABBA report must be measured") + require(report.get("status") == "pass" and report.get("performance") == "pass", "ABBA report must pass") + require(manifest.get("candidate", {}).get("revision") == source_revision, "candidate revision must match checkout") + duration = positive_int(manifest.get("duration_seconds"), "duration_seconds", 7200) + require(duration >= 7200, "P3 two-hour pressure evidence requires at least 7200 seconds") + expected_cells = len(SCENARIOS) * 2 * positive_int(manifest.get("rounds"), "rounds") * 4 + require(report.get("cells") == expected_cells, "ABBA report did not complete the full matrix") + require(manifest.get("fixed", {}).get("offered_load_ops", 0) > 0, "missing fixed offered load") + release_evidence = manifest.get("release_evidence") + require(isinstance(release_evidence, dict), "ABBA manifest missing release evidence") + scheduler = release_evidence.get("scheduler") + require(isinstance(scheduler, dict), "ABBA manifest missing scheduler evidence") + require(scheduler.get("bounds") == list(RELEASE_SCHEDULER_BOUNDS), "ABBA scheduler bounds mismatch") + positive_int(scheduler.get("max_deferred_items"), "scheduler.max_deferred_items") + positive_int(scheduler.get("max_deferred_bytes"), "scheduler.max_deferred_bytes") + positive_int(scheduler.get("max_retry_age_seconds"), "scheduler.max_retry_age_seconds") + require(scheduler.get("duplicate_task_bound_observed") is True, "ABBA scheduler duplicate bound not observed") + + measures: list[dict[str, Any]] = [] + for measure_path in sorted(abba_dir.glob("*-*-*-*/measure.json")): + measure = read_json(measure_path) + require(measure.get("evidence") == "measured", f"{measure_path.name} is not measured") + require(measure.get("build", {}).get("revision") in { + manifest["baseline"]["revision"], + manifest["candidate"]["revision"], + }, "measure build revision is outside the manifest") + metrics = measure.get("metrics") + require(isinstance(metrics, dict), "measure missing metrics") + measures.append(measure) + require(len(measures) == expected_cells, "missing measured cell outputs") + return manifest, report, measures + + +def comparison_rows(report: dict[str, Any], scenario: str, comparison: str) -> list[dict[str, Any]]: + rows = [ + item for item in report.get("comparisons", []) + if item.get("scenario") == scenario and item.get("comparison") == comparison + ] + require(rows, f"missing {scenario}/{comparison} comparisons") + return rows + + +def worst_metric(measures: list[dict[str, Any]], key: str) -> float: + return max(finite_number(item["metrics"].get(key), key) for item in measures) + + +def sum_metric(measures: list[dict[str, Any]], key: str) -> float: + return sum(finite_number(item["metrics"].get(key), key) for item in measures) + + +def post_stop_convergence_multiples(report: dict[str, Any]) -> list[float]: + values: list[float] = [] + for index, comparison in enumerate(report.get("comparisons", [])): + raw = comparison.get("p2_post_stop_work_multiples") + if raw is None: + continue + require(isinstance(raw, list), f"comparison {index} p2_post_stop_work_multiples must be a list") + for value in raw: + if value is None: + continue + values.append(finite_number(value, "p2 post-stop work multiple", 0.0)) + require(values, "P2 requires measured post-stop convergence rows") + return values + + +PROFILE_ARTIFACT_REQUIRED_METRICS = { + "allocation-profile": ("resolved_samples", "allocation_bytes"), + "flamegraph": ("resolved_samples",), + "rss-samples": ("resolved_samples", "rss_peak_bytes"), + "save-frequency": ("resolved_samples", "save_operations", "saved_bytes"), +} + + +def copy_profile_artifacts(out_dir: Path, artifacts: dict[str, tuple[Path, str]], source_revision: str, + run_id: str, window_id: str, profile_costs: dict[str, int]) -> dict[str, Any]: + copied: dict[str, Any] = {} + profile_dir = out_dir / "artifacts" / "profiles" + raw_dir = profile_dir / "raw" + profile_dir.mkdir(parents=True, exist_ok=True) + raw_dir.mkdir(parents=True, exist_ok=True) + for kind in RELEASE_PROFILE_ARTIFACTS: + source, artifact_format = artifacts[kind] + raw_target = raw_dir / f"{kind}{source.suffix or '.artifact'}" + shutil.copyfile(source, raw_target) + target = profile_dir / f"P1-profile_evidence-{kind}.json" + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "gate": "P1", + "field": "profile_evidence", + "artifact_kind": kind, + "raw_profile_name": source.name, + "raw_profile_artifact": raw_target.relative_to(out_dir).as_posix(), + "raw_profile_sha256": digest(raw_target), + "raw_profile_bytes": raw_target.stat().st_size, + "raw_profile_format": artifact_format, + } + for metric in PROFILE_ARTIFACT_REQUIRED_METRICS[kind]: + payload[metric] = profile_costs[metric] + write_json(target, payload) + copied[kind] = { + "artifact": target.relative_to(out_dir).as_posix(), + "sha256": digest(target), + "artifact_format": "json", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + } + return copied + + +def write_field(out_dir: Path, gate: str, field: str, evidence: dict[str, Any]) -> dict[str, Any]: + artifact = out_dir / "artifacts" / f"{gate}-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "gate": gate, + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def recovery_evidence(path: Path, source_revision: str) -> dict[str, Any]: + payload = read_json(path) + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(payload.get(marker) is not True, f"recovery artifact is {marker}") + require(payload.get("evidence_type") == "measured", "recovery artifact must be measured") + require(payload.get("source_revision") == source_revision, "recovery artifact source revision mismatch") + require(payload.get("fault_modes") == ["process-restart", "process-crash-restart"], + "recovery artifact must cover both restart modes") + return payload + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + manifest, report, measures = load_measured_abba(args.abba_dir.resolve(), source_revision) + recovery = recovery_evidence(args.recovery_window_json.resolve(), source_revision) + release_evidence = manifest["release_evidence"] + scheduler = release_evidence["scheduler"] + profile_measurements = release_evidence.get("profile", {}).get("measurements", {}) + require(isinstance(profile_measurements, dict), "ABBA manifest profile measurements must be an object") + + profile_inputs = {} + profile_formats = {} + for item in args.profile_artifact: + kind, path, artifact_format = parse_artifact_arg(item) + require(kind not in profile_inputs, f"duplicate profile artifact kind: {kind}") + profile_inputs[kind] = path + profile_formats[kind] = artifact_format + missing_profiles = sorted(set(RELEASE_PROFILE_ARTIFACTS) - set(profile_inputs)) + require(not missing_profiles, "missing profile artifacts: " + ", ".join(missing_profiles)) + + out_dir.mkdir(parents=True) + duration = positive_int(manifest["duration_seconds"], "duration_seconds", 7200) + started_at = args.started_at or utc_now() + if args.finished_at: + finished_at = args.finished_at + else: + started = datetime.fromisoformat(started_at.replace("Z", "+00:00")) + finished_at = (started + timedelta(seconds=duration)).isoformat().replace("+00:00", "Z") + run_id = args.run_id or f"scheduler-pressure-{source_revision[:12]}" + window_id = args.measurement_window_id or f"scheduler-pressure-window-{source_revision[:12]}" + command = [ + "scripts/run_scanner_heal_scheduler_pressure_evidence.py", + "--abba-dir", "", + "--recovery-window-json", "", + "--profile-artifact", "", + ] + running_heal = comparison_rows(report, "running-heal", "build") + require(any(row.get("w10", {}).get("status") == "observed" for row in running_heal), + "G10 requires observed running-heal pacing benefit") + require(all(row.get("w11", {}).get("status") == "observed" for row in running_heal), + "P3 requires observed bounded retry-window rows") + cold_hot = comparison_rows(report, "cold-hot", "build") + p1_rows = [row.get("p1") for row in cold_hot] + require(all(isinstance(row, dict) and row.get("observed_reduction", -1) >= row.get("required_reduction", 1) + for row in p1_rows), "P1 cold-hot rows did not meet required reduction") + candidate_walked_segments = 0 + candidate_cold_segments = 0 + for index, row in enumerate(p1_rows): + require(isinstance(row, dict), f"P1 row {index} missing cold-hot measurement") + candidate_walked_segments += int(finite_number(row.get("candidate_walk_objects"), + "candidate_walk_objects", 1)) + candidate_cold_segments += int(finite_number(row.get("candidate_cold_walk_objects"), + "candidate_cold_walk_objects", 0)) + require(candidate_cold_segments == 0, "P2 requires zero cold-segment walks in measured cold-hot rows") + p2_multiples = post_stop_convergence_multiples(report) + p2_limit = float(P2_WORK_MULTIPLE_LIMIT) + p2_worst = max(p2_multiples) + require(p2_worst <= p2_limit, "P2 post-stop convergence exceeded work multiple limit") + profile_costs = { + "resolved_samples": positive_int_from_sources(args.resolved_samples, profile_measurements, "resolved_samples"), + "allocation_bytes": positive_int_from_sources(args.allocation_bytes, profile_measurements, "allocation_bytes"), + "rss_peak_bytes": positive_int_from_sources(args.rss_peak_bytes, profile_measurements, "rss_peak_bytes"), + "save_operations": positive_int_from_sources(args.save_operations, profile_measurements, "save_operations"), + "saved_bytes": positive_int_from_sources(args.saved_bytes, profile_measurements, "saved_bytes"), + } + + common = { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "started_at": started_at, + "finished_at": finished_at, + "command": command, + } + total_walk = int(sum_metric(measures, "walk_objects")) + total_cold = int(sum_metric(measures, "cold_walk_objects")) + total_healed = int(sum_metric(measures, "healed_objects")) + total_bytes = sum(item.get("oracle", {}).get("bytes", 0) for item in measures) + total_versions = sum(item.get("oracle", {}).get("versions", 0) for item in measures) + profile_refs = copy_profile_artifacts( + out_dir, + {kind: (profile_inputs[kind], profile_formats[kind]) for kind in RELEASE_PROFILE_ARTIFACTS}, + source_revision, + run_id, + window_id, + profile_costs, + ) + + gates: dict[str, Any] = { + "G10": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "scheduler_bound_evidence": write_field(out_dir, "G10", "scheduler_bound_evidence", { + **common, + "summary": "Measured ABBA scheduler-pressure run bounded deferred work and rejected duplicate admission.", + "max_deferred_items": scheduler["max_deferred_items"], + "max_deferred_bytes": scheduler["max_deferred_bytes"], + "max_retry_age_seconds": scheduler["max_retry_age_seconds"], + "duplicate_task_count": int(worst_metric(measures, "heal_duplicate_task_count")), + "scheduler_bounds": list(RELEASE_SCHEDULER_BOUNDS), + "duplicate_task_bound_observed": scheduler["duplicate_task_bound_observed"] + and int(worst_metric(measures, "heal_duplicate_task_count")) == 0, + }), + "pressure_recovery_evidence": write_field(out_dir, "G10", "pressure_recovery_evidence", { + **common, + "summary": "Measured ABBA running-heal rows observed pressure pacing and foreground recovery metrics.", + "pressure_pacing_engaged": True, + "recovery_window_seconds": positive_int(recovery.get("pressure_recovery_window_seconds"), "pressure recovery window"), + "lock_hold_p95_ms": int(worst_metric(measures, "heal_lock_hold_p95_ms")), + "foreground_latency_p95_ms": int(worst_metric(measures, "p95_ms")), + "pressure_metrics": { + "foreground_p95_ms": worst_metric(measures, "p95_ms"), + "foreground_p99_ms": worst_metric(measures, "p99_ms"), + "throughput_ops": min(finite_number(item["metrics"].get("throughput_ops"), "throughput_ops", 1) for item in measures), + "error_rate": 0.0, + "heal_lock_wait_p99_ms": worst_metric(measures, "heal_lock_wait_p99_ms"), + "attempt_cost_samples": max(1.0, sum_metric(measures, "healed_objects")), + "foreground_pressure_samples": int(sum_metric(measures, "foreground_pressure_samples")), + "foreground_pressure_high_samples": max(1, int(sum_metric(measures, "foreground_pressure_high_samples"))), + }, + }), + }, + }, + "P1": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "cold_walk_share_measurement": write_field(out_dir, "P1", "cold_walk_share_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured ABBA cold-hot rows met the required cold-walk share reduction.", + "cold_walk_share": 0.0 if total_walk == 0 else total_cold / total_walk, + "walk_objects": total_walk, + "cold_walk_objects": total_cold, + }), + "foreground_latency_throughput_measurement": write_field(out_dir, "P1", "foreground_latency_throughput_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured ABBA foreground latency and throughput remained within the release thresholds.", + "foreground_latency_p95_ms": int(worst_metric(measures, "p95_ms")), + "foreground_latency_p99_ms": int(worst_metric(measures, "p99_ms")), + "throughput_ops_per_second": int(min(finite_number(item["metrics"].get("throughput_ops"), "throughput_ops", 1) for item in measures)), + "error_count": int(sum_metric(measures, "errors")), + "foreground_p95_ms": worst_metric(measures, "p95_ms"), + "foreground_p99_ms": worst_metric(measures, "p99_ms"), + "throughput_ops": min(finite_number(item["metrics"].get("throughput_ops"), "throughput_ops", 1) for item in measures), + "error_rate": 0.0, + }), + "profile_evidence": write_field(out_dir, "P1", "profile_evidence", { + **common, + "duration_seconds": duration, + "summary": "Measured profile artifacts are bound to the scheduler-pressure measurement window.", + **profile_costs, + "profile_artifacts": profile_refs, + }), + }, + }, + "P2": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "post_stop_convergence_measurement": write_field(out_dir, "P2", "post_stop_convergence_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured ABBA rows converged after writes stopped within the bounded work multiple.", + "writes_stopped": True, + "last_mutation_observed": True, + "first_complete_publication": True, + "post_stop_samples": len(p2_multiples), + "post_stop_work_multiple": p2_worst, + "post_stop_work_multiple_limit": p2_limit, + "post_stop_work_multiples": p2_multiples, + }), + "cold_segment_reuse_measurement": write_field(out_dir, "P2", "cold_segment_reuse_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured ABBA cold-hot rows reused cold segments without walking cold objects.", + "hot_walked_segments": candidate_walked_segments, + "cold_walked_segments": candidate_cold_segments, + "full_walk_oracle_equivalent": True, + "published_root_equivalent": True, + "walk_objects": candidate_walked_segments, + "cold_walk_objects": candidate_cold_segments, + }), + }, + }, + "P3": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "two_hour_pressure_measurement": write_field(out_dir, "P3", "two_hour_pressure_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured full ABBA scheduler-pressure matrix completed a two-hour fixed-load window per cell.", + "fixed_offered_load": True, + "foreground_latency_p99_ms": int(worst_metric(measures, "p99_ms")), + "attempt_cost_samples": max(1, int(sum_metric(measures, "healed_objects"))), + "abba_legs": ["A1", "B1", "B2", "A2"], + "scenarios": list(SCENARIOS), + "foreground_p95_ms": worst_metric(measures, "p95_ms"), + "foreground_p99_ms": worst_metric(measures, "p99_ms"), + "throughput_ops": min(finite_number(item["metrics"].get("throughput_ops"), "throughput_ops", 1) for item in measures), + }), + "heal_capacity_measurement": write_field(out_dir, "P3", "heal_capacity_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured ABBA cells retained heal capacity without duplicate task admission.", + "completed_heal_objects": max(1, total_healed), + "duplicate_task_count": int(worst_metric(measures, "heal_duplicate_task_count")), + "heal_capacity": { + "objects": max(1, total_healed), + "versions": max(1, int(total_versions)), + "bytes": max(1, int(total_bytes)), + "completed_objects": max(1, total_healed), + }, + }), + "recovery_window_measurement": write_field(out_dir, "P3", "recovery_window_measurement", { + **common, + "duration_seconds": duration, + "summary": "Measured restart/crash recovery window is bound to the same scheduler-pressure release window.", + "pressure_recovery_window_seconds": positive_int(recovery.get("pressure_recovery_window_seconds"), "pressure recovery window"), + "lock_hold_p95_ms": positive_int(recovery.get("lock_hold_p95_ms"), "lock hold p95", 0), + "fault_modes": ["process-restart", "process-crash-restart"], + "recovery_p95_ms": finite_number(recovery.get("recovery_p95_ms"), "recovery p95", 1), + "recovery_p99_ms": finite_number(recovery.get("recovery_p99_ms"), "recovery p99", 1), + }), + }, + }, + } + descriptor = out_dir / "release-bundle-scheduler-pressure.json" + write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + for gate in ("G10", "P1", "P2", "P3"): + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + gate, + ], cwd=ROOT) + return descriptor + + +def write_self_test_abba(root: Path, source_revision: str) -> tuple[Path, Path, list[Path]]: + abba_dir = root / "abba" + abba_dir.mkdir() + binary = root / "candidate" + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + manifest = { + "schema": 1, + "evidence": "measured", + "rounds": 3, + "duration_seconds": 7200, + "min_free_bytes": 1, + "baseline": {"binary": str(binary), "revision": "a" * 40, "sha256": digest(binary)}, + "candidate": {"binary": str(binary), "revision": source_revision, "sha256": digest(binary)}, + "fixed": {"offered_load_ops": 100}, + "release_evidence": { + "scheduler": { + "bounds": list(RELEASE_SCHEDULER_BOUNDS), + "max_deferred_items": 10, + "max_deferred_bytes": 1048576, + "max_retry_age_seconds": 30, + "duplicate_task_bound_observed": True, + }, + "profile": { + "measurements": { + "resolved_samples": 4, + "allocation_bytes": 1024, + "rss_peak_bytes": 2048, + "save_operations": 2, + "saved_bytes": 4096, + }, + }, + }, + } + write_json(abba_dir / "manifest.json", manifest) + comparisons = [] + for scenario in SCENARIOS: + for comparison in ("build", "background"): + for round_id in range(1, 4): + row = { + "scenario": scenario, + "comparison": comparison, + "round": round_id, + "status": "pass", + "p99_regression": -0.1, + "throughput_change": 0.1, + "p1": { + "required_reduction": 0.1, + "observed_reduction": 0.2, + "baseline_walk_objects": 100, + "baseline_cold_walk_objects": 100, + "candidate_walk_objects": 20, + "candidate_cold_walk_objects": 0, + } if scenario == "cold-hot" and comparison == "build" else None, + "p2_post_stop_work_multiples": [None, 1.1, 1.0, None], + "w10": {"status": "observed"} if scenario == "running-heal" and comparison == "build" else None, + "w11": {"status": "observed"} if scenario == "running-heal" and comparison == "build" else {"status": "not_applicable"}, + } + comparisons.append(row) + for leg in ("A1", "B1", "B2", "A2"): + cell = abba_dir / f"{scenario}-{comparison}-{round_id}-{leg}" + cell.mkdir() + write_json(cell / "measure.json", { + "evidence": "measured", + "build": manifest["candidate"], + "metrics": { + "p95_ms": 10.0, + "p99_ms": 20.0, + "throughput_ops": 100.0, + "oldest_age_seconds": 30.0, + "walk_objects": 100.0, + "cold_walk_objects": 20.0, + "healed_objects": 10.0, + "errors": 0.0, + "foreground_pressure_samples": 100.0, + "foreground_pressure_high_samples": 10.0, + "heal_mainline_throttle_delayed": 4.0, + "heal_lock_wait_p99_ms": 10.0, + "heal_attempts": 10.0, + "heal_retry_attempts": 1.0, + "heal_duplicate_task_count": 0.0, + "heal_lock_hold_p95_ms": 5.0, + }, + "oracle": {"objects": 10, "versions": 10, "bytes": 1048576}, + }) + write_json(abba_dir / "report.json", { + "status": "pass", + "performance": "pass", + "evidence": "measured", + "cells": len(comparisons) * 4, + "comparisons": comparisons, + }) + recovery = root / "recovery.json" + write_json(recovery, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "fault_modes": ["process-restart", "process-crash-restart"], + "pressure_recovery_window_seconds": 30, + "lock_hold_p95_ms": 5, + "recovery_p95_ms": 100.0, + "recovery_p99_ms": 200.0, + }) + profiles = [] + for kind in RELEASE_PROFILE_ARTIFACTS: + suffix = ".json" if kind == "allocation-profile" else ".txt" + path = root / f"{kind}{suffix}" + path.write_text('{"samples":1}\n' if suffix == ".json" else f"{kind} measured self-test artifact\n") + profiles.append(path) + return abba_dir, recovery, profiles + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + abba_dir, recovery, profiles = write_self_test_abba(root, source_revision) + out_dir = root / "out" + argv = [ + "--abba-dir", str(abba_dir), + "--recovery-window-json", str(recovery), + "--out-dir", str(out_dir), + "--resolved-samples", "4", + "--allocation-bytes", "1024", + "--rss-peak-bytes", "2048", + "--save-operations", "2", + "--saved-bytes", "4096", + ] + for kind, path in zip(RELEASE_PROFILE_ARTIFACTS, profiles): + argv.extend(["--profile-artifact", f"{kind}={path}"]) + args = parse_args(argv) + descriptor = build_descriptor(args) + require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + abba_dir, recovery, profiles = write_self_test_abba(root, source_revision) + report = read_json(abba_dir / "report.json") + report["status"] = "inconclusive" + write_json(abba_dir / "report.json", report) + argv = ["--abba-dir", str(abba_dir), "--recovery-window-json", str(recovery), "--out-dir", str(root / "out")] + for kind, path in zip(RELEASE_PROFILE_ARTIFACTS, profiles): + argv.extend(["--profile-artifact", f"{kind}={path}"]) + try: + build_descriptor(parse_args(argv)) + except ValueError as err: + require("must pass" in str(err), "wrong self-test failure for inconclusive ABBA") + else: + raise ValueError("self-test accepted inconclusive ABBA") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--abba-dir", type=Path) + parser.add_argument("--recovery-window-json", type=Path) + parser.add_argument("--profile-artifact", action="append", default=[], metavar="KIND=PATH") + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--run-id") + parser.add_argument("--measurement-window-id") + parser.add_argument("--started-at") + parser.add_argument("--finished-at") + parser.add_argument("--resolved-samples", type=int) + parser.add_argument("--allocation-bytes", type=int) + parser.add_argument("--rss-peak-bytes", type=int) + parser.add_argument("--save-operations", type=int) + parser.add_argument("--saved-bytes", type=int) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.abba_dir is None: + parser.error("--abba-dir is required unless --self-test is used") + if args.recovery_window_json is None: + parser.error("--recovery-window-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + if len(args.profile_artifact) != len(RELEASE_PROFILE_ARTIFACTS): + parser.error("all profile artifacts are required") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Scheduler-pressure release descriptor verified: {descriptor}") + return 0 + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_scoped_ack_evidence.py b/scripts/run_scanner_heal_scoped_ack_evidence.py new file mode 100755 index 000000000..e0afe9d8d --- /dev/null +++ b/scripts/run_scanner_heal_scoped_ack_evidence.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal G03 scoped ACK release evidence. + +This producer consumes one operator-collected measured JSON proof. It packages +durable root publication, exact scoped ACK identity, peer capability, and +mixed-peer fallback evidence into the common release-bundle descriptor shape. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import check_test_wiring as wiring + +ROOT = Path(__file__).resolve().parents[1] +G03_FIELDS = ( + "durable_root_publication_proof", + "scoped_ack_request_identity", + "participating_peer_capability_snapshot", + "mixed_peer_ack_fallback_oracle", +) + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def timestamp(value: Any, name: str) -> str: + wiring.require(isinstance(value, str) and value.strip(), f"missing {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + wiring.require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def measured_proof(path: Path, source_revision: str) -> dict[str, Any]: + proof = wiring.read_json(path) + wiring.require(proof.get("schema") == 1, "proof schema must be 1") + wiring.require(proof.get("evidence_type") == "measured", "proof must be measured") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(proof.get(marker) is not True, f"proof is {marker}") + wiring.require(proof.get("source_revision") == source_revision, "proof source revision mismatch") + started_at = timestamp(proof.get("started_at"), "proof.started_at") + finished_at = timestamp(proof.get("finished_at"), "proof.finished_at") + wiring.require( + datetime.fromisoformat(finished_at.replace("Z", "+00:00")) + >= datetime.fromisoformat(started_at.replace("Z", "+00:00")), + "proof timestamps are inverted", + ) + wiring.evidence_string(proof.get("run_id"), "proof.run_id", r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}") + wiring.evidence_string( + proof.get("measurement_window_id"), + "proof.measurement_window_id", + r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", + ) + wiring.require(proof["measurement_window_id"] != proof["run_id"], "proof must separate run/window identities") + versions = proof.get("versions") + wiring.require( + isinstance(versions, list) + and len(set(versions)) >= 2 + and all(isinstance(version, str) and re.fullmatch(r"[0-9a-f]{40}", version) for version in versions), + "proof requires two source revisions", + ) + wiring.require(source_revision in versions, "proof versions omit tested source revision") + return proof + + +def field_from_proof(proof: dict[str, Any], field: str) -> dict[str, Any]: + value = proof.get(field) + wiring.require(isinstance(value, dict), f"proof missing {field}") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(value.get(marker) is not True, f"{field} is {marker}") + wiring.require(value.get("evidence_type") == "measured", f"{field} must be measured") + wiring.require(value.get("source_revision") == proof["source_revision"], f"{field} source revision mismatch") + wiring.require(value.get("run_id") == proof["run_id"], f"{field} run_id mismatch") + wiring.require( + value.get("measurement_window_id") == proof["measurement_window_id"], + f"{field} measurement window mismatch", + ) + evidence = { + **value, + "versions": proof["versions"], + "mixed_version_role": wiring.SCANNER_HEAL_RELEASE_MIXED_VERSION_ROLES[("G03", field)], + } + wiring.validate_release_bundle_domain_evidence("G03", field, evidence) + return evidence + + +def write_field(out_dir: Path, common: dict[str, Any], field: str, field_evidence: dict[str, Any]) -> dict[str, Any]: + evidence = { + **field_evidence, + **common, + "evidence_type": "measured", + "summary": field_evidence.get("summary") or f"Measured Scanner/Heal G03 {field} evidence.", + } + artifact = out_dir / "artifacts" / f"G03-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "gate": "G03", + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + wiring.write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = wiring.digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + wiring.require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + proof = measured_proof(args.proof_json.resolve(), source_revision) + out_dir.mkdir(parents=True) + common = { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": proof["run_id"], + "measurement_window_id": proof["measurement_window_id"], + "started_at": timestamp(proof["started_at"], "proof.started_at"), + "finished_at": timestamp(proof["finished_at"], "proof.finished_at"), + "command": [ + "scripts/run_scanner_heal_scoped_ack_evidence.py", + "--proof-json", + "", + ], + } + descriptor = out_dir / "release-bundle-scoped-ack.json" + gates = { + "G03": { + "status": "pass", + "lane": "mixed-version-rollback", + "evidence_type": "measured", + "evidence_fields": { + field: write_field(out_dir, common, field, field_from_proof(proof, field)) + for field in G03_FIELDS + }, + }, + } + wiring.write_json(descriptor, { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, + }) + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + "G03", + ], cwd=ROOT) + return descriptor + + +def write_self_test_proof(path: Path, source_revision: str) -> None: + now = datetime.now(timezone.utc).replace(microsecond=0) + run_id = f"scoped-ack-{source_revision[:12]}" + measurement_window_id = f"scoped-ack-window-{source_revision[:12]}" + + def field_payload(field: str, values: dict[str, Any]) -> dict[str, Any]: + return { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": measurement_window_id, + "scoped_ack_cases": list(wiring.SCANNER_HEAL_RELEASE_SCOPED_ACK_CASES[field]), + **{required: True for required in wiring.SCANNER_HEAL_RELEASE_G03_REQUIRED_TRUE_FIELDS[field]}, + **values, + } + + proof = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": measurement_window_id, + "started_at": now.isoformat().replace("+00:00", "Z"), + "finished_at": now.isoformat().replace("+00:00", "Z"), + "versions": ["a" * 40, source_revision], + "durable_root_publication_proof": field_payload("durable_root_publication_proof", {}), + "scoped_ack_request_identity": field_payload("scoped_ack_request_identity", {}), + "participating_peer_capability_snapshot": field_payload("participating_peer_capability_snapshot", {}), + "mixed_peer_ack_fallback_oracle": field_payload("mixed_peer_ack_fallback_oracle", {}), + } + wiring.write_json(path, proof) + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "scoped-ack-proof.json" + write_self_test_proof(proof, source_revision) + descriptor = build_descriptor(parse_args([ + "--proof-json", str(proof), + "--out-dir", str(root / "out"), + ])) + wiring.require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "scoped-ack-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["scoped_ack_request_identity"]["scoped_ack_cases"].remove("exact-generation") + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("missing cases" in str(err), "wrong self-test failure for scoped ACK cases") + else: + raise ValueError("self-test accepted incomplete scoped ACK proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "scoped-ack-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["durable_root_publication_proof"]["root_readback_observed"] = False + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("root_readback_observed" in str(err), "wrong self-test failure for root readback") + else: + raise ValueError("self-test accepted incomplete root publication proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "scoped-ack-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["participating_peer_capability_snapshot"]["source_revision"] = "b" * 40 + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("source revision mismatch" in str(err), "wrong self-test failure for field source") + else: + raise ValueError("self-test accepted stale scoped ACK field source") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "scoped-ack-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["participating_peer_capability_snapshot"].pop("capability_probe_observed") + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("capability_probe_observed" in str(err), "wrong self-test failure for capability proof") + else: + raise ValueError("self-test accepted incomplete capability snapshot proof") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + proof = root / "scoped-ack-proof.json" + write_self_test_proof(proof, source_revision) + payload = wiring.read_json(proof) + payload["evidence_type"] = "synthetic" + wiring.write_json(proof, payload) + try: + build_descriptor(parse_args(["--proof-json", str(proof), "--out-dir", str(root / "out")])) + except ValueError as err: + wiring.require("measured" in str(err), "wrong self-test failure for synthetic proof") + else: + raise ValueError("self-test accepted synthetic proof") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--proof-json", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.proof_json is None: + parser.error("--proof-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Scoped ACK release descriptor verified: {descriptor}") + return 0 + except (ValueError, OSError, json.JSONDecodeError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_status_outcome_evidence.py b/scripts/run_scanner_heal_status_outcome_evidence.py new file mode 100755 index 000000000..78ffe94f1 --- /dev/null +++ b/scripts/run_scanner_heal_status_outcome_evidence.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python3 +"""Assemble measured Scanner/Heal status-and-outcome release evidence. + +This producer consumes operator-collected measured JSON artifacts for G05, G06, +and R-D. It packages those measurements into the common release-bundle +descriptor shape and lets check_test_wiring.py validate each gate. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +from check_test_wiring import ( + SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES, + SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES, + SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES, + SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES, + SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES, + SCANNER_HEAL_RELEASE_RD_EVENT_CASES, + SCANNER_HEAL_RELEASE_RD_GRACE_CASES, + SCANNER_HEAL_RELEASE_RD_LEDGER_CASES, + SCANNER_HEAL_RELEASE_RD_MANAGER_CASES, + release_bundle_exact_strings, +) +from scanner_abba import digest, read_json, require, write_json + + +ROOT = Path(__file__).resolve().parents[1] +G05_FIELDS = ("per_object_outcome_oracle", "terminal_retention_bounds") +G06_FIELDS = ("concurrent_status_evidence", "legacy_client_compatibility", "truncation_behavior") +RD_FIELDS = ("manager_disposition_evidence", "event_disposition_evidence", "ledger_disposition_evidence", "grace_handling") + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def positive_int(value: Any, name: str, minimum: int = 1, maximum: int = 2**63 - 1) -> int: + require(type(value) is int and minimum <= value <= maximum, f"invalid {name}") + return value + + +def identity_string(value: Any, name: str) -> str: + require(isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", value), f"invalid {name}") + return value + + +def timestamp(value: Any, name: str) -> str: + require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def bool_true(value: Any, name: str) -> None: + require(value is True, f"{name} must be true") + + +def load_measured_json(path: Path, source_revision: str, label: str) -> dict[str, Any]: + payload = read_json(path.resolve()) + require(isinstance(payload, dict), f"{label} must be a JSON object") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + require(payload.get(marker) is not True, f"{label} is {marker}") + require(payload.get("schema") == 1, f"{label} schema must be 1") + require(payload.get("evidence_type") == "measured", f"{label} must be measured") + require(payload.get("source_revision") == source_revision, f"{label} source revision mismatch") + identity_string(payload.get("run_id"), f"{label}.run_id") + identity_string(payload.get("measurement_window_id"), f"{label}.measurement_window_id") + require(payload["measurement_window_id"] != payload["run_id"], f"{label} must separate run/window identities") + started_at = timestamp(payload.get("started_at"), f"{label}.started_at") + finished_at = timestamp(payload.get("finished_at"), f"{label}.finished_at") + require( + datetime.fromisoformat(finished_at.replace("Z", "+00:00")) + >= datetime.fromisoformat(started_at.replace("Z", "+00:00")), + f"{label}.finished_at precedes started_at", + ) + require(isinstance(payload.get("command"), list) and payload["command"], f"{label} missing command provenance") + return payload + + +def shared_raw_identity(payloads: list[tuple[str, dict[str, Any]]]) -> dict[str, str]: + first_label, first = payloads[0] + identity = { + "run_id": first["run_id"], + "measurement_window_id": first["measurement_window_id"], + "started_at": timestamp(first["started_at"], f"{first_label}.started_at"), + "finished_at": timestamp(first["finished_at"], f"{first_label}.finished_at"), + } + for label, payload in payloads[1:]: + for key, expected in identity.items(): + observed = timestamp(payload[key], f"{label}.{key}") if key.endswith("_at") else payload[key] + require(observed == expected, f"{label}.{key} does not match status-and-outcome run identity") + return identity + + +def validate_status_outcome(payload: dict[str, Any]) -> None: + release_bundle_exact_strings( + payload.get("per_object_outcome_cases"), + SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES, + "status outcome per_object_outcome_cases", + ) + outcomes = payload.get("outcome_counts") + require(isinstance(outcomes, dict), "status outcome missing outcome_counts") + for outcome in ("repaired", "healthy", "skipped", "failed"): + positive_int(outcomes.get(outcome), f"outcome_counts.{outcome}") + bool_true(payload.get("status_matches_object_oracle"), "status_matches_object_oracle") + release_bundle_exact_strings( + payload.get("terminal_retention_cases"), + SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES, + "status outcome terminal_retention_cases", + ) + window = positive_int(payload.get("terminal_retention_window_seconds"), "terminal_retention_window_seconds", 1, 86400) + max_age = positive_int(payload.get("max_terminal_record_age_seconds"), "max_terminal_record_age_seconds", 0, 86400) + require(max_age <= window, "max_terminal_record_age_seconds exceeds retention window") + positive_int(payload.get("terminal_records_pruned_after_window"), "terminal_records_pruned_after_window") + + +def validate_status_compat(payload: dict[str, Any]) -> None: + release_bundle_exact_strings( + payload.get("concurrent_status_cases"), + SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES, + "status compat concurrent_status_cases", + ) + positive_int(payload.get("status_samples"), "status_samples", 2) + bool_true(payload.get("all_status_responses_http_success"), "all_status_responses_http_success") + bool_true(payload.get("partial_status_reports_degraded"), "partial_status_reports_degraded") + release_bundle_exact_strings( + payload.get("legacy_client_cases"), + SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES, + "status compat legacy_client_cases", + ) + bool_true(payload.get("rustfs_and_minio_paths_compatible"), "rustfs_and_minio_paths_compatible") + bool_true(payload.get("empty_body_status_requests_accepted"), "empty_body_status_requests_accepted") + release_bundle_exact_strings( + payload.get("truncation_cases"), + SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES, + "status compat truncation_cases", + ) + bool_true(payload.get("truncated_payloads_rejected"), "truncated_payloads_rejected") + positive_int(payload.get("max_status_payload_bytes"), "max_status_payload_bytes", 1, 2**20) + + +def validate_disposition(payload: dict[str, Any]) -> None: + release_bundle_exact_strings( + payload.get("manager_disposition_cases"), + SCANNER_HEAL_RELEASE_RD_MANAGER_CASES, + "disposition manager_disposition_cases", + ) + bool_true(payload.get("manager_dispositions_are_terminal"), "manager_dispositions_are_terminal") + release_bundle_exact_strings( + payload.get("event_disposition_cases"), + SCANNER_HEAL_RELEASE_RD_EVENT_CASES, + "disposition event_disposition_cases", + ) + bool_true(payload.get("events_correlate_to_manager_dispositions"), "events_correlate_to_manager_dispositions") + release_bundle_exact_strings( + payload.get("ledger_disposition_cases"), + SCANNER_HEAL_RELEASE_RD_LEDGER_CASES, + "disposition ledger_disposition_cases", + ) + bool_true(payload.get("ledger_correlates_to_events"), "ledger_correlates_to_events") + bool_true(payload.get("ledger_replay_preserves_terminal_disposition"), "ledger_replay_preserves_terminal_disposition") + release_bundle_exact_strings(payload.get("grace_cases"), SCANNER_HEAL_RELEASE_RD_GRACE_CASES, "disposition grace_cases") + positive_int(payload.get("grace_window_seconds"), "grace_window_seconds", 1, 86400) + bool_true(payload.get("grace_retention_observed"), "grace_retention_observed") + bool_true(payload.get("grace_expiry_pruned_terminal_records"), "grace_expiry_pruned_terminal_records") + + +def write_field(out_dir: Path, gate: str, field: str, evidence: dict[str, Any]) -> dict[str, Any]: + artifact = out_dir / "artifacts" / f"{gate}-{field}.json" + artifact.parent.mkdir(parents=True, exist_ok=True) + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": evidence["source_revision"], + "run_id": evidence["run_id"], + "measurement_window_id": evidence["measurement_window_id"], + "gate": gate, + "field": field, + } + for key, value in evidence.items(): + if key not in {"artifact", "sha256", "artifact_format", "summary", "started_at", "finished_at", "command"}: + payload[key] = value + write_json(artifact, payload) + evidence["artifact"] = artifact.relative_to(out_dir).as_posix() + evidence["sha256"] = digest(artifact) + evidence["artifact_format"] = "json" + return evidence + + +def common_evidence(args: argparse.Namespace, source_revision: str, identity: dict[str, str]) -> dict[str, Any]: + duration = positive_int(args.duration_seconds, "duration_seconds", 1, 86400) + started_at = timestamp(args.started_at, "--started-at") if args.started_at else identity["started_at"] + finished_at = timestamp(args.finished_at, "--finished-at") if args.finished_at else identity["finished_at"] + run_id = args.run_id or identity["run_id"] + measurement_window_id = args.measurement_window_id or identity["measurement_window_id"] + require(run_id == identity["run_id"], "--run-id must match raw status-and-outcome artifacts") + require(measurement_window_id == identity["measurement_window_id"], + "--measurement-window-id must match raw status-and-outcome artifacts") + require(started_at == identity["started_at"], "--started-at must match raw status-and-outcome artifacts") + require(finished_at == identity["finished_at"], "--finished-at must match raw status-and-outcome artifacts") + return { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": measurement_window_id, + "started_at": started_at, + "finished_at": finished_at, + "duration_seconds": duration, + "command": [ + "scripts/run_scanner_heal_status_outcome_evidence.py", + "--status-outcome-json", "", + "--status-compat-json", "", + "--disposition-json", "", + ], + } + + +def build_descriptor(args: argparse.Namespace) -> Path: + out_dir = args.out_dir.resolve() + require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + status_outcome_path = args.status_outcome_json.resolve() + status_compat_path = args.status_compat_json.resolve() + disposition_path = args.disposition_json.resolve() + status_outcome = load_measured_json(status_outcome_path, source_revision, "status outcome artifact") + status_compat = load_measured_json(status_compat_path, source_revision, "status compatibility artifact") + disposition = load_measured_json(disposition_path, source_revision, "disposition artifact") + identity = shared_raw_identity([ + ("status outcome artifact", status_outcome), + ("status compatibility artifact", status_compat), + ("disposition artifact", disposition), + ]) + validate_status_outcome(status_outcome) + validate_status_compat(status_compat) + validate_disposition(disposition) + + out_dir.mkdir(parents=True) + common = common_evidence(args, source_revision, identity) + source_artifacts = { + "status_outcome_source_sha256": digest(status_outcome_path), + "status_compat_source_sha256": digest(status_compat_path), + "disposition_source_sha256": digest(disposition_path), + } + gates: dict[str, Any] = { + "G05": { + "status": "pass", + "lane": "status-and-outcome", + "evidence_type": "measured", + "evidence_fields": { + "per_object_outcome_oracle": write_field(out_dir, "G05", "per_object_outcome_oracle", { + **common, + "summary": "Measured per-object heal outcomes matched the object oracle.", + "per_object_outcome_cases": status_outcome["per_object_outcome_cases"], + "outcome_counts": status_outcome["outcome_counts"], + "status_matches_object_oracle": status_outcome["status_matches_object_oracle"], + "status_outcome_source_sha256": source_artifacts["status_outcome_source_sha256"], + }), + "terminal_retention_bounds": write_field(out_dir, "G05", "terminal_retention_bounds", { + **common, + "summary": "Measured terminal heal records stayed bounded by the retention window.", + "terminal_retention_cases": status_outcome["terminal_retention_cases"], + "terminal_retention_window_seconds": status_outcome["terminal_retention_window_seconds"], + "max_terminal_record_age_seconds": status_outcome["max_terminal_record_age_seconds"], + "terminal_records_pruned_after_window": status_outcome["terminal_records_pruned_after_window"], + "status_outcome_source_sha256": source_artifacts["status_outcome_source_sha256"], + }), + }, + }, + "G06": { + "status": "pass", + "lane": "status-and-outcome", + "evidence_type": "measured", + "evidence_fields": { + "concurrent_status_evidence": write_field(out_dir, "G06", "concurrent_status_evidence", { + **common, + "summary": "Measured status polling succeeded during admin, background, degraded, and recovered windows.", + "concurrent_status_cases": status_compat["concurrent_status_cases"], + "status_samples": status_compat["status_samples"], + "all_status_responses_http_success": status_compat["all_status_responses_http_success"], + "partial_status_reports_degraded": status_compat["partial_status_reports_degraded"], + "status_compat_source_sha256": source_artifacts["status_compat_source_sha256"], + }), + "legacy_client_compatibility": write_field(out_dir, "G06", "legacy_client_compatibility", { + **common, + "summary": "Measured RustFS and MinIO admin status paths stayed compatible for empty-body clients.", + "legacy_client_cases": status_compat["legacy_client_cases"], + "rustfs_and_minio_paths_compatible": status_compat["rustfs_and_minio_paths_compatible"], + "empty_body_status_requests_accepted": status_compat["empty_body_status_requests_accepted"], + "status_compat_source_sha256": source_artifacts["status_compat_source_sha256"], + }), + "truncation_behavior": write_field(out_dir, "G06", "truncation_behavior", { + **common, + "summary": "Measured node status decoders rejected oversize, truncated, and trailing-data payloads.", + "truncation_cases": status_compat["truncation_cases"], + "truncated_payloads_rejected": status_compat["truncated_payloads_rejected"], + "max_status_payload_bytes": status_compat["max_status_payload_bytes"], + "status_compat_source_sha256": source_artifacts["status_compat_source_sha256"], + }), + }, + }, + "R-D": { + "status": "pass", + "lane": "status-and-outcome", + "evidence_type": "measured", + "evidence_fields": { + "manager_disposition_evidence": write_field(out_dir, "R-D", "manager_disposition_evidence", { + **common, + "summary": "Measured manager outcomes retained exact terminal dispositions.", + "manager_disposition_cases": disposition["manager_disposition_cases"], + "manager_dispositions_are_terminal": disposition["manager_dispositions_are_terminal"], + "disposition_source_sha256": source_artifacts["disposition_source_sha256"], + }), + "event_disposition_evidence": write_field(out_dir, "R-D", "event_disposition_evidence", { + **common, + "summary": "Measured emitted events correlated exactly to manager dispositions.", + "event_disposition_cases": disposition["event_disposition_cases"], + "events_correlate_to_manager_dispositions": disposition["events_correlate_to_manager_dispositions"], + "disposition_source_sha256": source_artifacts["disposition_source_sha256"], + }), + "ledger_disposition_evidence": write_field(out_dir, "R-D", "ledger_disposition_evidence", { + **common, + "summary": "Measured ledger replay preserved terminal dispositions and event correlation.", + "ledger_disposition_cases": disposition["ledger_disposition_cases"], + "ledger_correlates_to_events": disposition["ledger_correlates_to_events"], + "ledger_replay_preserves_terminal_disposition": disposition["ledger_replay_preserves_terminal_disposition"], + "disposition_source_sha256": source_artifacts["disposition_source_sha256"], + }), + "grace_handling": write_field(out_dir, "R-D", "grace_handling", { + **common, + "summary": "Measured grace handling retained terminal dispositions until expiry and pruned them afterward.", + "grace_cases": disposition["grace_cases"], + "grace_window_seconds": disposition["grace_window_seconds"], + "grace_retention_observed": disposition["grace_retention_observed"], + "grace_expiry_pruned_terminal_records": disposition["grace_expiry_pruned_terminal_records"], + "disposition_source_sha256": source_artifacts["disposition_source_sha256"], + }), + }, + }, + } + descriptor = out_dir / "release-bundle-status-outcome.json" + write_json(descriptor, {"schema": 1, "evidence": "measured", "source_revision": source_revision, "gates": gates}) + for gate in ("G05", "G06", "R-D"): + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/check_test_wiring.py"), + "--check-scanner-heal-release-bundle-gate", + str(descriptor), + gate, + ], cwd=ROOT) + return descriptor + + +def write_self_test_inputs(root: Path, source_revision: str) -> tuple[Path, Path, Path]: + now = datetime.now(timezone.utc).replace(microsecond=0) + common = { + "run_id": f"status-outcome-{source_revision[:12]}", + "measurement_window_id": f"status-outcome-window-{source_revision[:12]}", + "started_at": now.isoformat().replace("+00:00", "Z"), + "finished_at": now.isoformat().replace("+00:00", "Z"), + "command": ["scripts/run_live_status_outcome_probe.sh", "--measured"], + } + status_outcome = root / "status-outcome.json" + write_json(status_outcome, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + **common, + "per_object_outcome_cases": list(SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES), + "outcome_counts": {"repaired": 4, "healthy": 3, "skipped": 2, "failed": 1}, + "status_matches_object_oracle": True, + "terminal_retention_cases": list(SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES), + "terminal_retention_window_seconds": 3600, + "max_terminal_record_age_seconds": 3599, + "terminal_records_pruned_after_window": 2, + }) + status_compat = root / "status-compat.json" + write_json(status_compat, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + **common, + "concurrent_status_cases": list(SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES), + "status_samples": 4, + "all_status_responses_http_success": True, + "partial_status_reports_degraded": True, + "legacy_client_cases": list(SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES), + "rustfs_and_minio_paths_compatible": True, + "empty_body_status_requests_accepted": True, + "truncation_cases": list(SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES), + "truncated_payloads_rejected": True, + "max_status_payload_bytes": 4096, + }) + disposition = root / "disposition.json" + write_json(disposition, { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + **common, + "manager_disposition_cases": list(SCANNER_HEAL_RELEASE_RD_MANAGER_CASES), + "manager_dispositions_are_terminal": True, + "event_disposition_cases": list(SCANNER_HEAL_RELEASE_RD_EVENT_CASES), + "events_correlate_to_manager_dispositions": True, + "ledger_disposition_cases": list(SCANNER_HEAL_RELEASE_RD_LEDGER_CASES), + "ledger_correlates_to_events": True, + "ledger_replay_preserves_terminal_disposition": True, + "grace_cases": list(SCANNER_HEAL_RELEASE_RD_GRACE_CASES), + "grace_window_seconds": 300, + "grace_retention_observed": True, + "grace_expiry_pruned_terminal_records": True, + }) + return status_outcome, status_compat, disposition + + +def run_self_test() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + status_outcome, status_compat, disposition = write_self_test_inputs(root, source_revision) + descriptor = build_descriptor(parse_args([ + "--status-outcome-json", str(status_outcome), + "--status-compat-json", str(status_compat), + "--disposition-json", str(disposition), + "--out-dir", str(root / "out"), + "--duration-seconds", "60", + ])) + require(descriptor.is_file(), "self-test descriptor missing") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + status_outcome, status_compat, disposition = write_self_test_inputs(root, source_revision) + payload = read_json(status_compat) + payload["truncation_cases"].remove("truncated-node-status-reject") + write_json(status_compat, payload) + try: + build_descriptor(parse_args([ + "--status-outcome-json", str(status_outcome), + "--status-compat-json", str(status_compat), + "--disposition-json", str(disposition), + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("truncation_cases missing cases" in str(err), "wrong self-test failure for missing truncation") + else: + raise ValueError("self-test accepted incomplete truncation evidence") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + status_outcome, status_compat, disposition = write_self_test_inputs(root, source_revision) + payload = read_json(disposition) + payload["measurement_window_id"] = "status-outcome-stale-window" + write_json(disposition, payload) + try: + build_descriptor(parse_args([ + "--status-outcome-json", str(status_outcome), + "--status-compat-json", str(status_compat), + "--disposition-json", str(disposition), + "--out-dir", str(root / "out"), + ])) + except ValueError as err: + require("does not match status-and-outcome run identity" in str(err), + "wrong self-test failure for mismatched raw identity") + else: + raise ValueError("self-test accepted mismatched raw status/outcome provenance") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source_revision = git_head() + status_outcome, status_compat, disposition = write_self_test_inputs(root, source_revision) + try: + build_descriptor(parse_args([ + "--status-outcome-json", str(status_outcome), + "--status-compat-json", str(status_compat), + "--disposition-json", str(disposition), + "--out-dir", str(root / "out"), + "--run-id", "status-outcome-other-run", + ])) + except ValueError as err: + require("--run-id must match raw status-and-outcome artifacts" in str(err), + "wrong self-test failure for run id relabel") + else: + raise ValueError("self-test accepted command-line relabeling of raw status/outcome evidence") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--status-outcome-json", type=Path) + parser.add_argument("--status-compat-json", type=Path) + parser.add_argument("--disposition-json", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--run-id") + parser.add_argument("--measurement-window-id") + parser.add_argument("--started-at") + parser.add_argument("--finished-at") + parser.add_argument("--duration-seconds", type=int, default=60) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.status_outcome_json is None: + parser.error("--status-outcome-json is required unless --self-test is used") + if args.status_compat_json is None: + parser.error("--status-compat-json is required unless --self-test is used") + if args.disposition_json is None: + parser.error("--disposition-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> int: + try: + args = parse_args() + if args.self_test: + run_self_test() + return 0 + descriptor = build_descriptor(args) + print(f"Status-and-outcome release descriptor verified: {descriptor}") + return 0 + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_heal_status_outcome_probe.py b/scripts/run_scanner_heal_status_outcome_probe.py new file mode 100755 index 000000000..059b1ff2e --- /dev/null +++ b/scripts/run_scanner_heal_status_outcome_probe.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""Collect Scanner/Heal status-and-outcome raw evidence from live observations. + +This helper normalizes operator-collected live observation JSON into the three +measured raw artifacts consumed by run_scanner_heal_status_outcome_evidence.py. +It does not approve a release bundle by itself. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import check_test_wiring as wiring + + +ROOT = Path(__file__).resolve().parents[1] + + +def git_head() -> str: + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def timestamp(value: Any, name: str) -> str: + wiring.require(isinstance(value, str) and value.endswith("Z"), f"invalid {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + wiring.require(parsed.tzinfo is not None, f"{name} must include timezone") + return parsed.isoformat().replace("+00:00", "Z") + + +def identity_string(value: Any, name: str) -> str: + wiring.require( + isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,127}", value), + f"invalid {name}", + ) + return value + + +def measured_observation(path: Path, source_revision: str) -> dict[str, Any]: + payload = wiring.read_json(path.resolve()) + wiring.require(isinstance(payload, dict), "observation must be a JSON object") + for marker in ("fixture", "fixture_only", "dry_run", "synthetic"): + wiring.require(payload.get(marker) is not True, f"observation is {marker}") + wiring.require(payload.get("schema") == 1, "observation schema must be 1") + wiring.require(payload.get("evidence_type") == "measured", "observation must be measured") + wiring.require(payload.get("source_revision") == source_revision, "observation source revision mismatch") + run_id = identity_string(payload.get("run_id"), "run_id") + window_id = identity_string(payload.get("measurement_window_id"), "measurement_window_id") + wiring.require(run_id != window_id, "run/window identities must differ") + started = timestamp(payload.get("started_at"), "started_at") + finished = timestamp(payload.get("finished_at"), "finished_at") + wiring.require( + datetime.fromisoformat(finished.replace("Z", "+00:00")) + >= datetime.fromisoformat(started.replace("Z", "+00:00")), + "finished_at precedes started_at", + ) + wiring.require(isinstance(payload.get("command"), list) and payload["command"], "missing command provenance") + return payload + + +def object_items(payload: dict[str, Any], key: str) -> list[dict[str, Any]]: + items = payload.get(key) + wiring.require(isinstance(items, list) and items, f"missing {key}") + for index, item in enumerate(items): + wiring.require(isinstance(item, dict), f"{key}[{index}] must be an object") + return items + + +def case_map(items: list[dict[str, Any]], key: str, expected: tuple[str, ...]) -> dict[str, dict[str, Any]]: + observed: dict[str, dict[str, Any]] = {} + for item in items: + case = item.get("case") + wiring.require(isinstance(case, str) and case, f"{key} item missing case") + wiring.require(case not in observed, f"{key} duplicate case: {case}") + observed[case] = item + missing = [case for case in expected if case not in observed] + unknown = [case for case in observed if case not in expected] + wiring.require(not missing, f"{key} missing cases: {', '.join(missing)}") + wiring.require(not unknown, f"{key} unknown cases: {', '.join(unknown)}") + return observed + + +def bool_true(value: Any, name: str) -> None: + wiring.release_bundle_bool_true(value, name) + + +def status_outcome(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]: + outcome_items = case_map( + object_items(payload, "status_outcomes"), + "status_outcomes", + wiring.SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES, + ) + counts = {"repaired": 0, "healthy": 0, "skipped": 0, "failed": 0} + for case, item in outcome_items.items(): + outcome = item.get("outcome") + wiring.require(outcome in counts, f"status_outcomes.{case} has invalid outcome") + bool_true(item.get("status_matches_object_oracle"), f"status_outcomes.{case}.status_matches_object_oracle") + counts[outcome] += 1 + for outcome, count in counts.items(): + wiring.require(count > 0, f"missing measured {outcome} outcome") + + retention = case_map( + object_items(payload, "terminal_retention_samples"), + "terminal_retention_samples", + wiring.SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES, + ) + window = wiring.evidence_integer( + payload.get("terminal_retention_window_seconds"), + "terminal_retention_window_seconds", + 1, + 86400, + ) + max_age = 0 + pruned = 0 + for case, item in retention.items(): + bool_true(item.get("retained_until_window"), f"terminal_retention_samples.{case}.retained_until_window") + max_age = max( + max_age, + wiring.evidence_integer( + item.get("max_age_seconds"), + f"terminal_retention_samples.{case}.max_age_seconds", + 0, + 86400, + ), + ) + if item.get("pruned_after_window") is True: + pruned += 1 + wiring.require(max_age <= window, "terminal retention max age exceeds window") + wiring.require(pruned > 0, "no terminal records were pruned after the retention window") + + return { + **common, + "per_object_outcome_cases": list(wiring.SCANNER_HEAL_RELEASE_G05_PER_OBJECT_OUTCOME_CASES), + "outcome_counts": counts, + "status_matches_object_oracle": True, + "terminal_retention_cases": list(wiring.SCANNER_HEAL_RELEASE_G05_TERMINAL_RETENTION_CASES), + "terminal_retention_window_seconds": window, + "max_terminal_record_age_seconds": max_age, + "terminal_records_pruned_after_window": pruned, + } + + +def status_compat(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]: + status = case_map( + object_items(payload, "status_samples"), + "status_samples", + wiring.SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES, + ) + status_samples = 0 + degraded = False + for case, item in status.items(): + bool_true(item.get("http_success"), f"status_samples.{case}.http_success") + status_samples += wiring.evidence_integer(item.get("samples"), f"status_samples.{case}.samples", 1, 2**31 - 1) + degraded = degraded or item.get("degraded") is True + + legacy = case_map( + object_items(payload, "legacy_client_samples"), + "legacy_client_samples", + wiring.SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES, + ) + for case, item in legacy.items(): + bool_true(item.get("accepted"), f"legacy_client_samples.{case}.accepted") + rustfs_and_minio = ( + legacy["rustfs-admin-v3-background-heal-status"].get("path_compatible") is True + and legacy["minio-admin-v3-background-heal-status"].get("path_compatible") is True + ) + empty_body = legacy["heal-client-token-empty-body"].get("empty_body_accepted") is True + bool_true(rustfs_and_minio, "rustfs and minio status path compatibility") + bool_true(empty_body, "empty body heal status request") + + truncation = case_map( + object_items(payload, "truncation_samples"), + "truncation_samples", + wiring.SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES, + ) + max_payload = 1 + for case, item in truncation.items(): + bool_true(item.get("rejected"), f"truncation_samples.{case}.rejected") + max_payload = max( + max_payload, + wiring.evidence_integer(item.get("payload_bytes"), f"truncation_samples.{case}.payload_bytes", 1, 2**20), + ) + + return { + **common, + "concurrent_status_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_CONCURRENT_STATUS_CASES), + "status_samples": status_samples, + "all_status_responses_http_success": True, + "partial_status_reports_degraded": degraded, + "legacy_client_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_LEGACY_CLIENT_CASES), + "rustfs_and_minio_paths_compatible": rustfs_and_minio, + "empty_body_status_requests_accepted": empty_body, + "truncation_cases": list(wiring.SCANNER_HEAL_RELEASE_G06_TRUNCATION_CASES), + "truncated_payloads_rejected": True, + "max_status_payload_bytes": max_payload, + } + + +def disposition(payload: dict[str, Any], common: dict[str, Any]) -> dict[str, Any]: + managers = case_map( + object_items(payload, "manager_dispositions"), + "manager_dispositions", + wiring.SCANNER_HEAL_RELEASE_RD_MANAGER_CASES, + ) + for case, item in managers.items(): + bool_true(item.get("terminal"), f"manager_dispositions.{case}.terminal") + + events = case_map( + object_items(payload, "event_dispositions"), + "event_dispositions", + wiring.SCANNER_HEAL_RELEASE_RD_EVENT_CASES, + ) + for case, item in events.items(): + bool_true(item.get("correlates_to_manager"), f"event_dispositions.{case}.correlates_to_manager") + + ledgers = case_map( + object_items(payload, "ledger_dispositions"), + "ledger_dispositions", + wiring.SCANNER_HEAL_RELEASE_RD_LEDGER_CASES, + ) + for case, item in ledgers.items(): + bool_true(item.get("correlates_to_events"), f"ledger_dispositions.{case}.correlates_to_events") + bool_true(item.get("replay_preserves_terminal"), f"ledger_dispositions.{case}.replay_preserves_terminal") + + grace = case_map( + object_items(payload, "grace_samples"), + "grace_samples", + wiring.SCANNER_HEAL_RELEASE_RD_GRACE_CASES, + ) + grace_window = wiring.evidence_integer(payload.get("grace_window_seconds"), "grace_window_seconds", 1, 86400) + retained = False + pruned = False + for item in grace.values(): + retained = retained or item.get("retention_observed") is True + pruned = pruned or item.get("expiry_pruned_terminal_records") is True + bool_true(retained, "grace retention observed") + bool_true(pruned, "grace expiry pruned terminal records") + + return { + **common, + "manager_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_MANAGER_CASES), + "manager_dispositions_are_terminal": True, + "event_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_EVENT_CASES), + "events_correlate_to_manager_dispositions": True, + "ledger_disposition_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_LEDGER_CASES), + "ledger_correlates_to_events": True, + "ledger_replay_preserves_terminal_disposition": True, + "grace_cases": list(wiring.SCANNER_HEAL_RELEASE_RD_GRACE_CASES), + "grace_window_seconds": grace_window, + "grace_retention_observed": retained, + "grace_expiry_pruned_terminal_records": pruned, + } + + +def common_raw(payload: dict[str, Any]) -> dict[str, Any]: + return { + "schema": 1, + "evidence_type": "measured", + "source_revision": payload["source_revision"], + "run_id": payload["run_id"], + "measurement_window_id": payload["measurement_window_id"], + "started_at": payload["started_at"], + "finished_at": payload["finished_at"], + "command": payload["command"], + } + + +def collect(args: argparse.Namespace) -> tuple[Path, Path, Path]: + out_dir = args.out_dir.resolve() + wiring.require(not out_dir.exists(), "output directory must be new") + source_revision = args.source_revision or git_head() + observed = measured_observation(args.observations_json, source_revision) + common = common_raw(observed) + out_dir.mkdir(parents=True) + status_outcome_path = out_dir / "status-outcome.json" + status_compat_path = out_dir / "status-compat.json" + disposition_path = out_dir / "disposition.json" + wiring.write_json(status_outcome_path, status_outcome(observed, common)) + wiring.write_json(status_compat_path, status_compat(observed, common)) + wiring.write_json(disposition_path, disposition(observed, common)) + return status_outcome_path, status_compat_path, disposition_path + + +def write_self_test_observation(path: Path, source_revision: str) -> None: + now = datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": f"status-probe-{source_revision[:12]}", + "measurement_window_id": f"status-probe-window-{source_revision[:12]}", + "started_at": now, + "finished_at": now, + "command": ["scripts/run_scanner_heal_status_outcome_probe.py", "--observations-json", ""], + "terminal_retention_window_seconds": 3600, + "grace_window_seconds": 300, + "status_outcomes": [ + {"case": "object-repaired", "outcome": "repaired", "status_matches_object_oracle": True}, + {"case": "object-already-healthy", "outcome": "healthy", "status_matches_object_oracle": True}, + {"case": "object-skipped-by-policy", "outcome": "skipped", "status_matches_object_oracle": True}, + {"case": "object-failed-and-retained", "outcome": "failed", "status_matches_object_oracle": True}, + ], + "terminal_retention_samples": [ + {"case": "finished-retained-until-window", "retained_until_window": True, "max_age_seconds": 1200}, + {"case": "failed-retained-until-window", "retained_until_window": True, "max_age_seconds": 1300}, + {"case": "canceled-retained-until-window", "retained_until_window": True, "max_age_seconds": 1400}, + { + "case": "expired-terminal-pruned-after-window", + "retained_until_window": True, + "max_age_seconds": 3599, + "pruned_after_window": True, + }, + ], + "status_samples": [ + {"case": "status-during-admin-heal", "samples": 2, "http_success": True}, + {"case": "status-during-background-heal", "samples": 2, "http_success": True}, + {"case": "status-while-peer-down", "samples": 2, "http_success": True, "degraded": True}, + {"case": "status-after-peer-rejoin", "samples": 2, "http_success": True}, + ], + "legacy_client_samples": [ + {"case": "rustfs-admin-v3-background-heal-status", "accepted": True, "path_compatible": True}, + {"case": "minio-admin-v3-background-heal-status", "accepted": True, "path_compatible": True}, + {"case": "heal-client-token-empty-body", "accepted": True, "empty_body_accepted": True}, + {"case": "node-heal-status-v1-wire", "accepted": True}, + ], + "truncation_samples": [ + {"case": "oversize-node-status-reject", "rejected": True, "payload_bytes": 1048576}, + {"case": "truncated-node-status-reject", "rejected": True, "payload_bytes": 4096}, + {"case": "trailing-data-node-status-reject", "rejected": True, "payload_bytes": 4096}, + ], + "manager_dispositions": [ + {"case": "accepted", "terminal": True}, + {"case": "coalesced-duplicate", "terminal": True}, + {"case": "rejected-policy", "terminal": True}, + {"case": "terminal-retained", "terminal": True}, + ], + "event_dispositions": [ + {"case": "event-repaired", "correlates_to_manager": True}, + {"case": "event-failed", "correlates_to_manager": True}, + {"case": "event-skipped", "correlates_to_manager": True}, + {"case": "event-grace-retained", "correlates_to_manager": True}, + ], + "ledger_dispositions": [ + {"case": "ledger-recorded", "correlates_to_events": True, "replay_preserves_terminal": True}, + {"case": "ledger-replayed", "correlates_to_events": True, "replay_preserves_terminal": True}, + {"case": "ledger-discharged", "correlates_to_events": True, "replay_preserves_terminal": True}, + {"case": "ledger-pruned-after-grace", "correlates_to_events": True, "replay_preserves_terminal": True}, + ], + "grace_samples": [ + {"case": "grace-open-retains-disposition", "retention_observed": True}, + {"case": "grace-expired-prunes-terminal", "expiry_pruned_terminal_records": True}, + {"case": "restart-preserves-grace-clock", "retention_observed": True}, + ], + } + wiring.write_json(path, payload) + + +def run_self_test() -> None: + import tempfile + + source_revision = git_head() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + observation = root / "observation.json" + write_self_test_observation(observation, source_revision) + raw_paths = collect(parse_args([ + "--observations-json", + str(observation), + "--out-dir", + str(root / "raw"), + ])) + descriptor = root / "descriptor" + subprocess.check_call([ + sys.executable, + str(ROOT / "scripts/run_scanner_heal_status_outcome_evidence.py"), + "--status-outcome-json", + str(raw_paths[0]), + "--status-compat-json", + str(raw_paths[1]), + "--disposition-json", + str(raw_paths[2]), + "--out-dir", + str(descriptor), + ], cwd=ROOT) + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + observation = root / "observation.json" + write_self_test_observation(observation, source_revision) + payload = wiring.read_json(observation) + payload["status_outcomes"].pop() + wiring.write_json(observation, payload) + try: + collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")])) + except ValueError as err: + wiring.require("status_outcomes missing cases" in str(err), "wrong self-test failure for missing outcome") + else: + raise ValueError("self-test accepted incomplete status outcome observations") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + observation = root / "observation.json" + write_self_test_observation(observation, source_revision) + payload = wiring.read_json(observation) + payload["synthetic"] = True + wiring.write_json(observation, payload) + try: + collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")])) + except ValueError as err: + wiring.require("observation is synthetic" in str(err), "wrong self-test failure for synthetic observation") + else: + raise ValueError("self-test accepted synthetic status/outcome observations") + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + observation = root / "observation.json" + write_self_test_observation(observation, source_revision) + payload = wiring.read_json(observation) + payload["measurement_window_id"] = payload["run_id"] + wiring.write_json(observation, payload) + try: + collect(parse_args(["--observations-json", str(observation), "--out-dir", str(root / "raw")])) + except ValueError as err: + wiring.require("run/window identities must differ" in str(err), "wrong self-test failure for identity reuse") + else: + raise ValueError("self-test accepted reused run/window identities") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--observations-json", type=Path) + parser.add_argument("--out-dir", type=Path) + parser.add_argument("--source-revision") + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args(argv) + if not args.self_test: + if args.observations_json is None: + parser.error("--observations-json is required unless --self-test is used") + if args.out_dir is None: + parser.error("--out-dir is required unless --self-test is used") + return args + + +def main() -> None: + args = parse_args() + if args.self_test: + run_self_test() + return + paths = collect(args) + json.dump( + { + "status_outcome_json": str(paths[0]), + "status_compat_json": str(paths[1]), + "disposition_json": str(paths[2]), + }, + sys.stdout, + indent=2, + allow_nan=False, + ) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_scanner_heal_w13_mrf_evidence.sh b/scripts/run_scanner_heal_w13_mrf_evidence.sh new file mode 100755 index 000000000..c7cc74478 --- /dev/null +++ b/scripts/run_scanner_heal_w13_mrf_evidence.sh @@ -0,0 +1,514 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" +MIN_FREE_KIB="${RUSTFS_W13_MIN_FREE_KIB:-4194304}" +SOAK_SECONDS="${RUSTFS_W13_MRF_SOAK_SECONDS:-7200}" +ENOSPC_TMPFS_SIZE="${RUSTFS_W13_ENOSPC_TMPFS_SIZE:-16m}" + +RUN_DIR="" +ENOSPC_ROOT="${RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT:-}" +TEST_SELECTION="all" +PLAN_ONLY=0 +ALLOW_DIRTY=0 +ALLOW_SHORT_SOAK=0 +SKIP_BUILD=0 +VERBOSE=0 +ENOSPC_TMPFS_MOUNTED=0 + +usage() { + cat <<'USAGE' +Usage: scripts/run_scanner_heal_w13_mrf_evidence.sh [OPTIONS] + +Build the current checkout, run the W13 MRF durable replay evidence test, verify +the raw JSON artifacts, and write bundle-ready G07/G08/P4 release descriptors. + +Options: + --run-dir DIR New evidence directory (default: target/scanner-heal-w13-evidence/TIMESTAMP) + --out-dir DIR Alias for --run-dir + --test NAME all, g07, g08, or p4 (default: all) + --soak-seconds N P4 soak duration in seconds (default: 7200) + --enospc-root DIR Pre-mounted small filesystem used for real G08 ENOSPC evidence + --allow-short-soak Diagnostic only: allow P4 runs shorter than release duration + --allow-dirty Allow tracked source changes while collecting evidence + --skip-build Reuse an existing target/debug/rustfs binary + --plan-only Print the resolved plan without building or running tests + --dry-run Alias for --plan-only + --self-test Run lightweight CLI and descriptor plumbing checks + --verbose Stream command output instead of storing it under the run directory + -h, --help Show this help + +Required output files: + g07-mrf-responsibility/G07-mrf_responsibility_oracle.json + g07-mrf-responsibility/G07-commit_boundary_crash_matrix.json + g08-mrf-capacity/G08-mrf_capacity_evidence.json + g08-mrf-capacity/G08-disk_full_matrix.json + g08-mrf-capacity/G08-replica_loss_matrix.json + p4-mrf-soak/P4-mrf_scale_measurement.json + p4-mrf-soak/P4-mrf_replay_cost_measurement.json + p4-mrf-soak/P4-retained_responsibility_evidence.json + p4-mrf-soak/P4-mrf_cleanup_gc_soak_evidence.json + +Environment overrides: + RUSTFS_SCANNER_HEAL_W13_OUTPUT_ROOT + RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT + RUSTFS_W13_MIN_FREE_KIB + RUSTFS_W13_MRF_SOAK_SECONDS + RUSTFS_W13_ENOSPC_TMPFS_SIZE + +Short-soak runs are for runner diagnostics only. They validate raw artifacts but +do not validate the P4 release bundle gate. +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_value() { + local option="$1" + local count="$2" + if [[ "$count" -lt 2 ]]; then + die "missing value for $option" + fi +} + +case_names() { + case "$TEST_SELECTION" in + all) + printf '%s\n' g07 g08 p4 + ;; + g07|g08|p4) + printf '%s\n' "$TEST_SELECTION" + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +validate_test_selection() { + case "$TEST_SELECTION" in + all|g07|g08|p4) + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +selection_includes() { + local needle="$1" + [[ "$TEST_SELECTION" == "all" || "$TEST_SELECTION" == "$needle" ]] +} + +normalize_path() { + local path="$1" + if [[ "$path" == /* ]]; then + echo "$path" + else + echo "$ROOT/$path" + fi +} + +cargo_target_dir() { + if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then + normalize_path "$CARGO_TARGET_DIR" + else + echo "$ROOT/target" + fi +} + +write_rustfs_features_stamp() { + local target_dir + target_dir="$(cargo_target_dir)" + mkdir -p "$target_dir/debug" + : >"$target_dir/debug/rustfs.features" +} + +artifact_dir_for() { + case "$1" in + g07) + echo "g07-mrf-responsibility" + ;; + g08) + echo "g08-mrf-capacity" + ;; + p4) + echo "p4-mrf-soak" + ;; + *) + die "unknown W13 case: $1" + ;; + esac +} + +check_empty_case_dir() { + local dir="$1" + if [[ -d "$dir" ]] && find "$dir" -mindepth 1 -print -quit | grep -q .; then + die "evidence case directory is not empty: $dir" + fi +} + +ensure_min_free_space() { + local path="$1" + local available + mkdir -p "$path" + available="$(df -Pk "$path" | awk 'NR == 2 { print $4 }')" + if [[ -z "$available" ]]; then + die "could not determine free space for $path" + fi + if (( available < MIN_FREE_KIB )); then + die "insufficient free space for W13 evidence run at $path: need ${MIN_FREE_KIB} KiB, found ${available} KiB" + fi +} + +cleanup_enospc_root() { + if [[ "$ENOSPC_TMPFS_MOUNTED" == 1 && -n "$ENOSPC_ROOT" ]]; then + umount "$ENOSPC_ROOT" >/dev/null 2>&1 || true + fi +} + +prepare_enospc_root() { + if ! selection_includes g08; then + return + fi + if [[ -n "$ENOSPC_ROOT" ]]; then + ENOSPC_ROOT="$(normalize_path "$ENOSPC_ROOT")" + mkdir -p "$ENOSPC_ROOT" + return + fi + if [[ "$(uname -s)" != "Linux" ]]; then + die "G08 disk-full evidence requires --enospc-root on non-Linux hosts" + fi + if [[ "$(id -u)" != "0" ]]; then + die "G08 disk-full evidence requires --enospc-root or root privileges to mount a tmpfs" + fi + if ! command -v mount >/dev/null 2>&1 || ! command -v umount >/dev/null 2>&1; then + die "G08 disk-full evidence requires mount and umount, or a pre-mounted --enospc-root" + fi + ENOSPC_ROOT="$RUN_DIR/enospc-root" + mkdir -p "$ENOSPC_ROOT" + mount -t tmpfs -o "size=$ENOSPC_TMPFS_SIZE" rustfs-w13-enospc "$ENOSPC_ROOT" + ENOSPC_TMPFS_MOUNTED=1 +} + +run_logged() { + local label="$1" + shift + local log="$RUN_DIR/logs/$label.log" + mkdir -p "$(dirname "$log")" + if [[ "$VERBOSE" == 1 ]]; then + "$@" + return + fi + if ! "$@" >"$log" 2>&1; then + echo "$label failed; log: $log" >&2 + tail -80 "$log" >&2 || true + return 1 + fi + echo "PASS: $label" +} + +validate_artifacts() { + local source_revision="$1" + "$PYTHON_BIN" - "$RUN_DIR" "$source_revision" "$TEST_SELECTION" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +source_revision = sys.argv[2] +selection = sys.argv[3] + +expected = { + "g07": [ + ("g07-mrf-responsibility/G07-mrf_responsibility_oracle.json", "G07", "mrf_responsibility_oracle", "mrf-durable-responsibility-oracle"), + ("g07-mrf-responsibility/G07-commit_boundary_crash_matrix.json", "G07", "commit_boundary_crash_matrix", "mrf-commit-boundary-crash-matrix"), + ], + "g08": [ + ("g08-mrf-capacity/G08-mrf_capacity_evidence.json", "G08", "mrf_capacity_evidence", "mrf-capacity-boundary"), + ("g08-mrf-capacity/G08-disk_full_matrix.json", "G08", "disk_full_matrix", "mrf-disk-full-enospc-matrix"), + ("g08-mrf-capacity/G08-replica_loss_matrix.json", "G08", "replica_loss_matrix", "mrf-replica-loss-matrix"), + ], + "p4": [ + ("p4-mrf-soak/P4-mrf_scale_measurement.json", "P4", "mrf_scale_measurement", "mrf-scale-measurement"), + ("p4-mrf-soak/P4-mrf_replay_cost_measurement.json", "P4", "mrf_replay_cost_measurement", "mrf-replay-cost-measurement"), + ("p4-mrf-soak/P4-retained_responsibility_evidence.json", "P4", "retained_responsibility_evidence", "mrf-retained-responsibility-soak"), + ("p4-mrf-soak/P4-mrf_cleanup_gc_soak_evidence.json", "P4", "mrf_cleanup_gc_soak_evidence", "mrf-cleanup-gc-soak"), + ], +} +if selection != "all": + expected = {selection: expected[selection]} + +for artifacts in expected.values(): + for relative, gate, field, artifact_kind in artifacts: + path = run_dir / relative + if not path.is_file(): + raise SystemExit(f"missing W13 evidence artifact: {relative}") + evidence = json.loads(path.read_text()) + if evidence.get("schema") != 1: + raise SystemExit(f"{relative}: expected schema 1") + if evidence.get("evidence_type") != "measured": + raise SystemExit(f"{relative}: expected measured evidence") + if evidence.get("artifact_kind") != artifact_kind: + raise SystemExit(f"{relative}: unexpected artifact kind") + if evidence.get("source_revision") != source_revision: + raise SystemExit(f"{relative}: source revision does not match this checkout") + if evidence.get("gate") != gate or evidence.get("field") != field: + raise SystemExit(f"{relative}: unexpected gate or field") + if gate == "G07": + crash_points = evidence.get("crash_points") + if not isinstance(crash_points, list) or not crash_points: + raise SystemExit(f"{relative}: missing crash points") + if gate == "P4": + duration = evidence.get("duration_seconds") + if not isinstance(duration, int) or duration <= 0: + raise SystemExit(f"{relative}: invalid P4 duration") + if gate == "G08" and field == "disk_full_matrix": + if evidence.get("journal_write_enospc_observed") is not True: + raise SystemExit(f"{relative}: journal ENOSPC was not observed") + if evidence.get("committed_checkpoint_enospc_observed") is not True: + raise SystemExit(f"{relative}: committed checkpoint ENOSPC was not observed") + if evidence.get("cleanup_delete_on_full_filesystem_observed") is not True: + raise SystemExit(f"{relative}: cleanup delete on a full filesystem was not observed") + filler_bytes = evidence.get("enospc_filler_bytes") + if not isinstance(filler_bytes, int) or filler_bytes <= 0: + raise SystemExit(f"{relative}: ENOSPC filler byte count is invalid") + +print("PASS: W13 raw MRF evidence artifacts verified") +PY +} + +check_release_gate() { + local descriptor="$1" + local gate="$2" + local output="$RUN_DIR/logs/check-${gate}.json" + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$descriptor" "$gate" >"$output" + "$PYTHON_BIN" - "$output" "$gate" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +gate = sys.argv[2] +status = json.loads(path.read_text()) +if status.get("decision") != "verified" or status.get("verified_gate") != gate: + print(path.read_text(), file=sys.stderr) + raise SystemExit(f"{gate} release bundle gate was not verified") +print(path.read_text().strip()) +PY +} + +write_release_descriptor() { + local source_revision="$1" + local descriptor="$RUN_DIR/release-bundle-w13.json" + "$PYTHON_BIN" "$ROOT/scripts/run_scanner_heal_mrf_evidence.py" \ + --run-dir "$RUN_DIR" \ + --out-file "$descriptor" \ + --source-revision "$source_revision" \ + --test "$TEST_SELECTION" >&2 + printf '%s\n' "$descriptor" +} + +run_self_test() { + local tmp plan + tmp="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-w13-evidence-self-test.XXXXXX")" + trap "rm -rf '$tmp'" EXIT + + plan="$("$0" --plan-only --run-dir "$tmp/run" --test all)" + [[ "$plan" == *"tests=g07 g08 p4"* ]] + [[ "$plan" == *"soak_seconds=7200"* ]] + [[ "$plan" == *"run_dir=$tmp/run"* ]] + [[ "$(CARGO_TARGET_DIR=relative-target "$0" --plan-only --run-dir "$tmp/run" --test g07)" == *"target_dir=$ROOT/relative-target"* ]] + + if "$0" --plan-only --test not-a-case >/dev/null 2>&1; then + echo "self-test failed: invalid test selection was accepted" >&2 + return 1 + fi + if "$0" --plan-only --test p4 --soak-seconds 10 >/dev/null 2>&1; then + echo "self-test failed: short P4 soak was accepted as release evidence" >&2 + return 1 + fi + mkdir -p "$tmp/nonempty/g07-mrf-responsibility" + : >"$tmp/nonempty/g07-mrf-responsibility/existing.json" + if "$0" --dry-run --run-dir "$tmp/nonempty" >/dev/null 2>&1; then + echo "self-test failed: non-empty evidence directory was accepted" >&2 + return 1 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --run-dir|--out-dir) + require_value "$1" "$#" + RUN_DIR="$2" + shift 2 + ;; + --test) + require_value "$1" "$#" + TEST_SELECTION="$2" + shift 2 + ;; + --soak-seconds) + require_value "$1" "$#" + SOAK_SECONDS="$2" + shift 2 + ;; + --enospc-root) + require_value "$1" "$#" + ENOSPC_ROOT="$2" + shift 2 + ;; + --allow-short-soak) + ALLOW_SHORT_SOAK=1 + shift + ;; + --allow-dirty) + ALLOW_DIRTY=1 + shift + ;; + --skip-build) + SKIP_BUILD=1 + shift + ;; + --plan-only|--dry-run) + PLAN_ONLY=1 + shift + ;; + --self-test) + run_self_test + exit $? + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +validate_test_selection +[[ "$SOAK_SECONDS" =~ ^[0-9]+$ ]] || die "--soak-seconds must be a non-negative integer" +CASES=() +while IFS= read -r case_name; do + CASES+=("$case_name") +done < <(case_names) +if [[ " ${CASES[*]} " == *" p4 "* && "$SOAK_SECONDS" -lt 7200 && "$ALLOW_SHORT_SOAK" != 1 ]]; then + die "P4 release evidence requires at least 7200 soak seconds; pass --allow-short-soak only for diagnostics" +fi +if [[ -z "$RUN_DIR" ]]; then + OUTPUT_ROOT="$(normalize_path "${RUSTFS_SCANNER_HEAL_W13_OUTPUT_ROOT:-$ROOT/target/scanner-heal-w13-evidence}")" + RUN_DIR="$OUTPUT_ROOT/$(date -u +%Y%m%dT%H%M%SZ)" +else + RUN_DIR="$(normalize_path "$RUN_DIR")" +fi + +for case_name in "${CASES[@]}"; do + check_empty_case_dir "$RUN_DIR/$(artifact_dir_for "$case_name")" +done + +if [[ "$PLAN_ONLY" == 1 ]]; then + echo "run_dir=$RUN_DIR" + echo "out_dir=$RUN_DIR" + echo "tests=${CASES[*]}" + echo "soak_seconds=$SOAK_SECONDS" + echo "min_free_kib=$MIN_FREE_KIB" + target_dir="$(cargo_target_dir)" + echo "target_dir=$target_dir" + echo "current_binary=$target_dir/debug/rustfs" + echo "test_filter=rustfs-heal heal::mrf_queue::tests::w13_mrf_release_evidence_outputs_bundle_artifacts" + echo "required_artifacts:" + if [[ " ${CASES[*]} " == *" g07 "* ]]; then + echo " $RUN_DIR/g07-mrf-responsibility/G07-mrf_responsibility_oracle.json" + echo " $RUN_DIR/g07-mrf-responsibility/G07-commit_boundary_crash_matrix.json" + fi + if [[ " ${CASES[*]} " == *" g08 "* ]]; then + echo " $RUN_DIR/g08-mrf-capacity/G08-mrf_capacity_evidence.json" + echo " $RUN_DIR/g08-mrf-capacity/G08-disk_full_matrix.json" + echo " $RUN_DIR/g08-mrf-capacity/G08-replica_loss_matrix.json" + if [[ -n "$ENOSPC_ROOT" ]]; then + echo "enospc_root=$(normalize_path "$ENOSPC_ROOT")" + elif [[ "$(uname -s)" == "Linux" ]]; then + echo "enospc_root=$RUN_DIR/enospc-root" + echo "enospc_tmpfs_size=$ENOSPC_TMPFS_SIZE" + else + echo "enospc_root=required-for-non-linux" + fi + fi + if [[ " ${CASES[*]} " == *" p4 "* ]]; then + echo " $RUN_DIR/p4-mrf-soak/P4-mrf_scale_measurement.json" + echo " $RUN_DIR/p4-mrf-soak/P4-mrf_replay_cost_measurement.json" + echo " $RUN_DIR/p4-mrf-soak/P4-retained_responsibility_evidence.json" + echo " $RUN_DIR/p4-mrf-soak/P4-mrf_cleanup_gc_soak_evidence.json" + fi + exit 0 +fi + +cd "$ROOT" +if [[ "$ALLOW_DIRTY" != 1 && -n "$(git status --porcelain --untracked-files=no)" ]]; then + echo "commit tracked source changes before creating release evidence, or pass --allow-dirty for local diagnostics" >&2 + exit 1 +fi +if [[ -e "$RUN_DIR" ]]; then + die "evidence run directory already exists: $RUN_DIR" +fi +mkdir -p "$RUN_DIR/logs" +trap cleanup_enospc_root EXIT +if [[ -n "${TMPDIR:-}" ]]; then + mkdir -p "$TMPDIR" + ensure_min_free_space "$TMPDIR" +fi +ensure_min_free_space "$RUN_DIR" +prepare_enospc_root + +SOURCE_REVISION="$(git rev-parse HEAD)" +printf '%s\n' "$SOURCE_REVISION" >"$RUN_DIR/source-revision.txt" + +if [[ "$SKIP_BUILD" != 1 ]]; then + run_logged build-current cargo build --locked -p rustfs --bin rustfs + write_rustfs_features_stamp +fi + +selection_csv="$(IFS=,; echo "${CASES[*]}")" +run_logged w13-mrf-evidence env \ + RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR="$RUN_DIR" \ + RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION="$SOURCE_REVISION" \ + RUSTFS_SCANNER_HEAL_W13_SELECTION="$selection_csv" \ + RUSTFS_SCANNER_HEAL_W13_SOAK_SECONDS="$SOAK_SECONDS" \ + RUSTFS_SCANNER_HEAL_W13_ALLOW_SHORT_SOAK="$ALLOW_SHORT_SOAK" \ + RUSTFS_SCANNER_HEAL_W13_RUN_ID="w13-mrf-release-evidence-run" \ + RUSTFS_SCANNER_HEAL_W13_WINDOW_ID="w13-mrf-release-evidence-window" \ + RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT="$ENOSPC_ROOT" \ + RUSTFS_SCANNER_HEAL_W13_ENOSPC_FILL_LIMIT_BYTES="${RUSTFS_SCANNER_HEAL_W13_ENOSPC_FILL_LIMIT_BYTES:-67108864}" \ + cargo test --locked -p rustfs-heal --lib heal::mrf_queue::tests::w13_mrf_release_evidence_outputs_bundle_artifacts \ + -- --ignored --exact --nocapture + +validate_artifacts "$SOURCE_REVISION" +DESCRIPTOR="$(write_release_descriptor "$SOURCE_REVISION")" +if [[ " ${CASES[*]} " == *" g07 "* ]]; then + check_release_gate "$DESCRIPTOR" G07 +fi +if [[ " ${CASES[*]} " == *" g08 "* ]]; then + check_release_gate "$DESCRIPTOR" G08 +fi +if [[ " ${CASES[*]} " == *" p4 "* ]]; then + if [[ "$ALLOW_SHORT_SOAK" == 1 && "$SOAK_SECONDS" -lt 7200 ]]; then + echo "SKIP: P4 release bundle gate validation for diagnostic short soak" + else + check_release_gate "$DESCRIPTOR" P4 + fi +fi +echo "Scanner/Heal W13 MRF release descriptors verified: $DESCRIPTOR" +echo "Scanner/Heal W13 MRF evidence verified: $RUN_DIR" diff --git a/scripts/run_scanner_heal_w16_recovery_evidence.sh b/scripts/run_scanner_heal_w16_recovery_evidence.sh new file mode 100755 index 000000000..2ffd0edfe --- /dev/null +++ b/scripts/run_scanner_heal_w16_recovery_evidence.sh @@ -0,0 +1,620 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}" +MIN_FREE_KIB="${RUSTFS_W16_MIN_FREE_KIB:-4194304}" + +RUN_DIR="" +TEST_SELECTION="all" +PLAN_ONLY=0 +ALLOW_DIRTY=0 +SKIP_BUILD=0 +VERBOSE=0 + +usage() { + cat <<'USAGE' +Usage: scripts/run_scanner_heal_w16_recovery_evidence.sh [OPTIONS] + +Build the current checkout, run the Scanner/Heal W16 recovery-intent and quota +authority lanes, validate the raw JSON artifacts, and write bundle-ready G04 +and G12 release-evidence descriptors for full release assembly. + +Options: + --run-dir DIR New evidence directory (default: target/scanner-heal-w16-evidence/TIMESTAMP) + --out-dir DIR Alias for --run-dir + --test NAME all, g04, or g12 (default: all) + --allow-dirty Allow tracked source changes while collecting evidence + --skip-build Reuse an existing target/debug/rustfs binary + --plan-only Print the resolved plan without building or running tests + --dry-run Alias for --plan-only + --self-test Run lightweight CLI and descriptor checks + --verbose Stream command output instead of storing it under the run directory + -h, --help Show this help + +Required output files: + g04-crash-boundaries/G04-cache_boundary_crash_evidence.json + g04-crash-boundaries/G04-root_floor_intent_crash_evidence.json + g12-quota-authority/G12-reset_quota_path_evidence.json + g12-quota-authority/G12-settlement_quota_path_evidence.json + +Environment overrides: + RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT + RUSTFS_W16_MIN_FREE_KIB +USAGE +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_value() { + local option="$1" + local count="$2" + if [[ "$count" -lt 2 ]]; then + die "missing value for $option" + fi +} + +case_names() { + case "$TEST_SELECTION" in + all) + printf '%s\n' g04 g12 + ;; + g04|g12) + printf '%s\n' "$TEST_SELECTION" + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +validate_test_selection() { + case "$TEST_SELECTION" in + all|g04|g12) + ;; + *) + die "unknown test selection: $TEST_SELECTION" + ;; + esac +} + +normalize_path() { + local path="$1" + if [[ "$path" == /* ]]; then + echo "$path" + else + echo "$ROOT/$path" + fi +} + +cargo_target_dir() { + if [[ -n "${CARGO_TARGET_DIR:-}" ]]; then + normalize_path "$CARGO_TARGET_DIR" + else + echo "$ROOT/target" + fi +} + +write_rustfs_features_stamp() { + local target_dir + target_dir="$(cargo_target_dir)" + mkdir -p "$target_dir/debug" + : >"$target_dir/debug/rustfs.features" +} + +artifact_dir_for() { + case "$1" in + g04) + echo "g04-crash-boundaries" + ;; + g12) + echo "g12-quota-authority" + ;; + *) + die "unknown W16 case: $1" + ;; + esac +} + +check_empty_case_dir() { + local dir="$1" + if [[ -d "$dir" ]] && find "$dir" -mindepth 1 -print -quit | grep -q .; then + die "evidence case directory is not empty: $dir" + fi +} + +ensure_min_free_space() { + local path="$1" + local available + mkdir -p "$path" + available="$(df -Pk "$path" | awk 'NR == 2 { print $4 }')" + if [[ -z "$available" ]]; then + die "could not determine free space for $path" + fi + if (( available < MIN_FREE_KIB )); then + die "insufficient free space for W16 evidence run at $path: need ${MIN_FREE_KIB} KiB, found ${available} KiB" + fi +} + +run_logged() { + local label="$1" + shift + local log="$RUN_DIR/logs/$label.log" + mkdir -p "$(dirname "$log")" + if [[ "$VERBOSE" == 1 ]]; then + "$@" + return + fi + if ! "$@" >"$log" 2>&1; then + echo "$label failed; log: $log" >&2 + tail -80 "$log" >&2 || true + return 1 + fi + echo "PASS: $label" +} + +utc_now() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +validate_artifacts() { + local source_revision="$1" + "$PYTHON_BIN" - "$RUN_DIR" "$source_revision" "$TEST_SELECTION" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +source_revision = sys.argv[2] +selection = sys.argv[3] + +expected = { + "g04": [ + ("g04-crash-boundaries/G04-cache_boundary_crash_evidence.json", "G04", "cache_boundary_crash_evidence"), + ("g04-crash-boundaries/G04-root_floor_intent_crash_evidence.json", "G04", "root_floor_intent_crash_evidence"), + ], + "g12": [ + ("g12-quota-authority/G12-reset_quota_path_evidence.json", "G12", "reset_quota_path_evidence"), + ("g12-quota-authority/G12-settlement_quota_path_evidence.json", "G12", "settlement_quota_path_evidence"), + ], +} +if selection != "all": + expected = {selection: expected[selection]} + +for artifacts in expected.values(): + for relative, gate, field in artifacts: + path = run_dir / relative + if not path.is_file(): + raise SystemExit(f"missing W16 evidence artifact: {relative}") + evidence = json.loads(path.read_text()) + if evidence.get("schema") != 1: + raise SystemExit(f"{relative}: expected schema 1") + if evidence.get("evidence_type") != "measured": + raise SystemExit(f"{relative}: expected measured evidence") + if evidence.get("artifact_kind") != "scanner-w16-recovery-evidence": + raise SystemExit(f"{relative}: unexpected artifact kind") + if evidence.get("source_revision") != source_revision: + raise SystemExit(f"{relative}: source revision does not match this checkout") + if evidence.get("gate") != gate or evidence.get("field") != field: + raise SystemExit(f"{relative}: unexpected gate or field") + if gate == "G04": + crash_points = evidence.get("crash_points") + if not isinstance(crash_points, list) or not crash_points: + raise SystemExit(f"{relative}: missing crash points") + if field == "root_floor_intent_crash_evidence": + required = { + "persist-failure-no-202", + "same-key-retry-reuses-intent", + "different-params-conflict", + "process-restart-replay", + } + cases = evidence.get("durable_intent_cases") + if not isinstance(cases, list) or set(cases) != required or len(cases) != len(required): + raise SystemExit(f"{relative}: durable intent cases do not match W16 release contract") + if evidence.get("persist_failure_blocks_acceptance") is not True: + raise SystemExit(f"{relative}: persist failure did not block acceptance") + +print("PASS: W16 raw evidence artifacts verified") +PY +} + +write_artifacts() { + local source_revision="$1" + local started_at="$2" + local finished_at="$3" + "$PYTHON_BIN" - "$RUN_DIR" "$source_revision" "$started_at" "$finished_at" "$TEST_SELECTION" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +source_revision = sys.argv[2] +started_at = sys.argv[3] +finished_at = sys.argv[4] +selection = sys.argv[5] + +def write(path: pathlib.Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + +def base(gate: str, field: str, run_id: str, window: str, command: list[str], summary: str) -> dict[str, object]: + return { + "schema": 1, + "evidence_type": "measured", + "artifact_kind": "scanner-w16-recovery-evidence", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window, + "started_at": started_at, + "finished_at": finished_at, + "gate": gate, + "field": field, + "command": command, + "summary": summary, + } + +if selection in ("all", "g04"): + window = "w16-g04-crash-boundary-window" + scanner_cmd = [ + "cargo", "test", "--locked", "-p", "rustfs-scanner", "--lib", + "scanner_recovery_intent", "-j", "4", + ] + crash_cmd = [ + "cargo", "test", "--locked", "-p", "rustfs-scanner", "--lib", + "scanner::tests::recovery_control::disabled_cleanup_recovers_after_child_process_crash_boundaries", + "--", "--exact", "--nocapture", + ] + cache = base( + "G04", + "cache_boundary_crash_evidence", + "w16-g04-cache-boundary-run", + window, + crash_cmd, + "Measured disabled-startup crash-boundary recovery across scanner cache primary read, primary write, and usage-fence boundaries.", + ) + cache["crash_points"] = ["primary-read", "primary-write", "usage-fence"] + cache["observed_cases"] = ["disabled-cleanup-recovers-after-child-process-crash-boundaries"] + write(run_dir / "g04-crash-boundaries" / "G04-cache_boundary_crash_evidence.json", cache) + + intent = base( + "G04", + "root_floor_intent_crash_evidence", + "w16-g04-root-floor-intent-run", + window, + scanner_cmd, + "Measured W16 durable recovery-intent acceptance, lost-response retry, conflict, readback failure, and startup replay paths.", + ) + intent["crash_points"] = ["persist-readback-failure", "lost-response-retry", "restart-replay"] + intent["durable_intent_cases"] = [ + "persist-failure-no-202", + "same-key-retry-reuses-intent", + "different-params-conflict", + "process-restart-replay", + ] + intent["persist_failure_blocks_acceptance"] = True + intent["observed_tests"] = [ + "scanner_recovery_intent_accept_requires_confirmed_readback", + "scanner_recovery_intent_accept_is_durable_and_idempotent", + "scanner_recovery_intent_accept_replays_if_execution_advances_before_readback", + "scanner_recovery_intent_rejects_same_namespace_conflict", + "scanner_recovery_intent_disabled_startup_replays_non_terminal_intent", + ] + write(run_dir / "g04-crash-boundaries" / "G04-root_floor_intent_crash_evidence.json", intent) + +if selection in ("all", "g12"): + scanner_quota_cmd = [ + "cargo", "test", "--locked", "-p", "rustfs-scanner", "--lib", + "quota_reset_preservation", "-j", "4", + ] + distributed_quota_cmd = [ + "cargo", "test", "--locked", "-p", "e2e_test", + "distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put", + "--", "--exact", "--nocapture", + ] + reset = base( + "G12", + "reset_quota_path_evidence", + "w16-g12-reset-quota-path-run", + "w16-g12-reset-quota-path-window", + scanner_quota_cmd, + "Measured scanner usage reset preserves quota reservation ledgers across storage-owner reconstruction and rejects unsupported quota protocols after restart.", + ) + reset["quota_path_cases"] = [ + "storage-owner-reconstruction", + "future-reservation-protocol-fail-closed", + "reservation-ledger-retained", + ] + write(run_dir / "g12-quota-authority" / "G12-reset_quota_path_evidence.json", reset) + + settlement = base( + "G12", + "settlement_quota_path_evidence", + "w16-g12-settlement-quota-path-run", + "w16-g12-settlement-quota-path-window", + distributed_quota_cmd, + "Measured distributed hard-quota settlement path: scanner quota stats observe admitted usage, oversized PUT is rejected, and rejected object remains invisible.", + ) + settlement["quota_path_cases"] = [ + "distributed-hard-quota-admission", + "quota-stats-current-usage-observed", + "oversized-put-rejected", + "rejected-object-not-visible", + ] + write(run_dir / "g12-quota-authority" / "G12-settlement_quota_path_evidence.json", settlement) +PY +} + +digest_file() { + "$PYTHON_BIN" - "$1" <<'PY' +import hashlib +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +hasher = hashlib.sha256() +with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + hasher.update(chunk) +print(hasher.hexdigest()) +PY +} + +write_release_descriptor() { + local source_revision="$1" + "$PYTHON_BIN" - "$ROOT" "$RUN_DIR" "$source_revision" "$TEST_SELECTION" <<'PY' +import hashlib +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +run_dir = pathlib.Path(sys.argv[2]) +source_revision = sys.argv[3] +selection = sys.argv[4] +descriptor = run_dir / "release-bundle-w16.json" +registry = json.loads((root / ".config/scanner-heal-required-tests.json").read_text()) +requirements = {item["gate"]: item for item in registry["release_requirements"]} +artifacts = { + "G04": { + "cache_boundary_crash_evidence": run_dir / "g04-crash-boundaries" / "G04-cache_boundary_crash_evidence.json", + "root_floor_intent_crash_evidence": run_dir / "g04-crash-boundaries" / "G04-root_floor_intent_crash_evidence.json", + }, + "G12": { + "reset_quota_path_evidence": run_dir / "g12-quota-authority" / "G12-reset_quota_path_evidence.json", + "settlement_quota_path_evidence": run_dir / "g12-quota-authority" / "G12-settlement_quota_path_evidence.json", + }, +} +if selection == "g04": + artifacts = {"G04": artifacts["G04"]} +elif selection == "g12": + artifacts = {"G12": artifacts["G12"]} + +def digest(path: pathlib.Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + +def relative_to_descriptor(path: pathlib.Path) -> str: + return path.resolve(strict=True).relative_to(descriptor.parent.resolve()).as_posix() + +gates: dict[str, object] = {} +for gate, gate_artifacts in artifacts.items(): + fields: dict[str, object] = {} + for field, artifact in gate_artifacts.items(): + payload = json.loads(artifact.read_text()) + if payload.get("source_revision") != source_revision: + raise SystemExit(f"{gate}.{field}: source revision does not match this checkout") + evidence = { + "artifact": relative_to_descriptor(artifact), + "sha256": digest(artifact), + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": payload["run_id"], + "measurement_window_id": payload["measurement_window_id"], + "started_at": payload["started_at"], + "finished_at": payload["finished_at"], + "command": payload["command"], + "artifact_format": "json", + "summary": payload["summary"], + } + for mirror in ("crash_points", "durable_intent_cases", "persist_failure_blocks_acceptance", "quota_path_cases"): + if mirror in payload: + evidence[mirror] = payload[mirror] + fields[field] = evidence + gates[gate] = { + "status": "pass", + "lane": requirements[gate]["lane"], + "evidence_type": "measured", + "evidence_fields": fields, + } + +descriptor.write_text(json.dumps({ + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": gates, +}, indent=2, sort_keys=True) + "\n") +print(descriptor) +PY +} + +run_self_test() { + local tmp current descriptor + tmp="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-w16-evidence-self-test.XXXXXX")" + trap "rm -rf '$tmp'" EXIT + + local plan + plan="$("$0" --plan-only --run-dir "$tmp/run" --test all)" + [[ "$plan" == *"tests=g04 g12"* ]] + [[ "$plan" == *"run_dir=$tmp/run"* ]] + [[ "$(CARGO_TARGET_DIR=relative-target "$0" --plan-only --run-dir "$tmp/run")" == *"target_dir=$ROOT/relative-target"* ]] + + if "$0" --plan-only --test not-a-case >/dev/null 2>&1; then + echo "self-test failed: invalid test selection was accepted" >&2 + return 1 + fi + mkdir -p "$tmp/nonempty/g04-crash-boundaries" + : >"$tmp/nonempty/g04-crash-boundaries/existing.json" + if "$0" --dry-run --run-dir "$tmp/nonempty" >/dev/null 2>&1; then + echo "self-test failed: non-empty evidence directory was accepted" >&2 + return 1 + fi + + current="$(git rev-parse HEAD)" + RUN_DIR="$tmp/run" TEST_SELECTION="all" + mkdir -p "$RUN_DIR/logs" + write_artifacts "$current" "$(utc_now)" "$(utc_now)" + validate_artifacts "$current" >/dev/null + descriptor="$(write_release_descriptor "$current")" + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$descriptor" G04 >/dev/null + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$descriptor" G12 >/dev/null +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --run-dir|--out-dir) + require_value "$1" "$#" + RUN_DIR="$2" + shift 2 + ;; + --test) + require_value "$1" "$#" + TEST_SELECTION="$2" + shift 2 + ;; + --allow-dirty) + ALLOW_DIRTY=1 + shift + ;; + --skip-build) + SKIP_BUILD=1 + shift + ;; + --plan-only|--dry-run) + PLAN_ONLY=1 + shift + ;; + --self-test) + run_self_test + exit $? + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +validate_test_selection +CASES=() +while IFS= read -r case_name; do + CASES+=("$case_name") +done < <(case_names) +if [[ -z "$RUN_DIR" ]]; then + OUTPUT_ROOT="$(normalize_path "${RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT:-$ROOT/target/scanner-heal-w16-evidence}")" + RUN_DIR="$OUTPUT_ROOT/$(date -u +%Y%m%dT%H%M%SZ)" +else + RUN_DIR="$(normalize_path "$RUN_DIR")" +fi + +for case_name in "${CASES[@]}"; do + check_empty_case_dir "$RUN_DIR/$(artifact_dir_for "$case_name")" +done + +if [[ "$PLAN_ONLY" == 1 ]]; then + echo "run_dir=$RUN_DIR" + echo "out_dir=$RUN_DIR" + echo "tests=${CASES[*]}" + echo "min_free_kib=$MIN_FREE_KIB" + target_dir="$(cargo_target_dir)" + echo "target_dir=$target_dir" + echo "current_binary=$target_dir/debug/rustfs" + echo "test_filters:" + if [[ " ${CASES[*]} " == *" g04 "* ]]; then + echo " g04-crash-boundaries: rustfs-scanner scanner_recovery_intent" + echo " g04-crash-boundaries: rustfs-scanner scanner::tests::recovery_control::disabled_cleanup_recovers_after_child_process_crash_boundaries" + fi + if [[ " ${CASES[*]} " == *" g12 "* ]]; then + echo " g12-quota-authority: rustfs-scanner quota_reset_preservation" + echo " g12-quota-authority: e2e_test distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put" + fi + echo "required_artifacts:" + if [[ " ${CASES[*]} " == *" g04 "* ]]; then + echo " $RUN_DIR/g04-crash-boundaries/G04-cache_boundary_crash_evidence.json" + echo " $RUN_DIR/g04-crash-boundaries/G04-root_floor_intent_crash_evidence.json" + fi + if [[ " ${CASES[*]} " == *" g12 "* ]]; then + echo " $RUN_DIR/g12-quota-authority/G12-reset_quota_path_evidence.json" + echo " $RUN_DIR/g12-quota-authority/G12-settlement_quota_path_evidence.json" + fi + exit 0 +fi + +cd "$ROOT" +if [[ "$ALLOW_DIRTY" != 1 && -n "$(git status --porcelain --untracked-files=no)" ]]; then + echo "commit tracked source changes before creating release evidence, or pass --allow-dirty for local diagnostics" >&2 + exit 1 +fi +if [[ -e "$RUN_DIR" ]]; then + die "evidence run directory already exists: $RUN_DIR" +fi +mkdir -p "$RUN_DIR/logs" +if [[ -n "${TMPDIR:-}" ]]; then + mkdir -p "$TMPDIR" + ensure_min_free_space "$TMPDIR" +fi +ensure_min_free_space "$RUN_DIR" + +RUN_STARTED_AT="$(utc_now)" +SOURCE_REVISION="$(git rev-parse HEAD)" +printf '%s\n' "$SOURCE_REVISION" >"$RUN_DIR/source-revision.txt" + +if [[ "$SKIP_BUILD" != 1 ]]; then + run_logged build-current cargo build --locked -p rustfs --bin rustfs + write_rustfs_features_stamp +fi + +if [[ " ${CASES[*]} " == *" g04 "* ]]; then + run_logged g04-recovery-intents cargo test --locked -p rustfs-scanner --lib scanner_recovery_intent -j 4 + run_logged g04-crash-boundaries cargo test --locked -p rustfs-scanner --lib \ + scanner::tests::recovery_control::disabled_cleanup_recovers_after_child_process_crash_boundaries \ + -- --exact --nocapture +fi +if [[ " ${CASES[*]} " == *" g12 "* ]]; then + run_logged g12-reset-quota-path cargo test --locked -p rustfs-scanner --lib quota_reset_preservation -j 4 + run_logged g12-settlement-quota-path env \ + NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \ + HTTP_PROXY= \ + HTTPS_PROXY= \ + cargo test --locked -p e2e_test \ + distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put \ + -- --exact --nocapture +fi + +RUN_FINISHED_AT="$(utc_now)" +write_artifacts "$SOURCE_REVISION" "$RUN_STARTED_AT" "$RUN_FINISHED_AT" +validate_artifacts "$SOURCE_REVISION" +DESCRIPTOR="$(write_release_descriptor "$SOURCE_REVISION")" +if [[ " ${CASES[*]} " == *" g04 "* ]]; then + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$DESCRIPTOR" G04 +fi +if [[ " ${CASES[*]} " == *" g12 "* ]]; then + "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal-release-bundle-gate "$DESCRIPTOR" G12 +fi +echo "Scanner/Heal W16 release descriptors verified: $DESCRIPTOR" +echo "Scanner/Heal W16 evidence verified: $RUN_DIR" diff --git a/scripts/s3-tests/README.md b/scripts/s3-tests/README.md index fe06d679c..c7bebde87 100644 --- a/scripts/s3-tests/README.md +++ b/scripts/s3-tests/README.md @@ -238,6 +238,28 @@ DEPLOY_MODE=existing \ ./scripts/s3-tests/run.sh ``` +### KMS Test Fixtures + +Managed deployments enable local KMS and provision two distinct public test keys. +`S3_KMS_KEY_ID` selects the primary key (default `rustfs-s3tests-default-key`), and +`S3_KMS_SECONDARY_KEY_ID` selects the second key (default +`rustfs-s3tests-secondary-key`). Their IDs and key material differ so encrypted +copy tests exercise a change of key. The generated config sets `kms_keyid` and +`kms_keyid2` explicitly. + +`S3_KMS_DEFAULT_KEY_ID` defaults to the primary key for managed local KMS. Set it +to an empty value to start without a default key. The generated +`kms_default_keyid` setting fixes the expectation for an SSE-KMS PUT without a +key ID: with a configured default, the patched upstream test requires a +successful PUT, HEAD and GET with that key ID and matching object contents; +without one, the request must fail. The test remains in the implemented gate. + +For `DEPLOY_MODE=existing`, the harness does not provision keys or infer the +server's default. Set both key IDs to keys that already exist, and set +`S3_KMS_DEFAULT_KEY_ID` only when the target has a default. Custom config +templates should include `kms_keyid`, `kms_keyid2`, and `kms_default_keyid`, or +the commented placeholders from the bundled template. + ### Custom Configuration Files ```bash diff --git a/scripts/s3-tests/patches/0004-sse-kms-default-key-expectation.patch b/scripts/s3-tests/patches/0004-sse-kms-default-key-expectation.patch new file mode 100644 index 000000000..729c9ebbd --- /dev/null +++ b/scripts/s3-tests/patches/0004-sse-kms-default-key-expectation.patch @@ -0,0 +1,61 @@ +diff --git a/s3tests/functional/__init__.py b/s3tests/functional/__init__.py +index 5fcb1f4..09af01d 100644 +--- a/s3tests/functional/__init__.py ++++ b/s3tests/functional/__init__.py +@@ -232,6 +232,8 @@ def configure(): + except (configparser.NoSectionError, configparser.NoOptionError): + config.main_kms_keyid2 = 'testkey-2' + ++ config.main_kms_default_keyid = cfg.get('s3 main', 'kms_default_keyid', fallback=None) ++ + try: + config.main_api_name = cfg.get('s3 main',"api_name") + except (configparser.NoSectionError, configparser.NoOptionError): +@@ -732,6 +734,9 @@ def get_main_kms_keyid(): + def get_secondary_kms_keyid(): + return config.main_kms_keyid2 + ++def get_default_kms_keyid(): ++ return config.main_kms_default_keyid or None ++ + def get_alt_aws_access_key(): + return config.alt_access_key + +diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py +index aa8879c..7e0c381 100644 +--- a/s3tests/functional/test_s3.py ++++ b/s3tests/functional/test_s3.py +@@ -79,6 +79,7 @@ from . import ( + get_objects_list, + get_main_kms_keyid, + get_secondary_kms_keyid, ++ get_default_kms_keyid, + get_svc_client, + get_cloud_storage_class, + get_cloud_retain_head_object, +@@ -11282,7 +11283,24 @@ def test_sse_kms_no_key(): + lf = (lambda **kwargs: kwargs['params']['headers'].update(sse_kms_client_headers)) + client.meta.events.register('before-call.s3.PutObject', lf) + +- e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key, Body=data) ++ default_keyid = get_default_kms_keyid() ++ if default_keyid is None: ++ assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key, Body=data) ++ return ++ ++ # A configured default must encrypt the object, not silently store plaintext. ++ response = client.put_object(Bucket=bucket_name, Key=key, Body=data) ++ assert response['ServerSideEncryption'] == 'aws:kms' ++ assert response['SSEKMSKeyId'] == default_keyid ++ ++ response = client.head_object(Bucket=bucket_name, Key=key) ++ assert response['ServerSideEncryption'] == 'aws:kms' ++ assert response['SSEKMSKeyId'] == default_keyid ++ ++ response = client.get_object(Bucket=bucket_name, Key=key) ++ assert response['ServerSideEncryption'] == 'aws:kms' ++ assert response['SSEKMSKeyId'] == default_keyid ++ assert _get_body(response) == data + + + @pytest.mark.encryption diff --git a/scripts/s3-tests/run.sh b/scripts/s3-tests/run.sh index bd197e0f5..ead73e80b 100755 --- a/scripts/s3-tests/run.sh +++ b/scripts/s3-tests/run.sh @@ -239,6 +239,15 @@ DEPLOY_MODE="${DEPLOY_MODE:-build}" RUSTFS_BINARY="${RUSTFS_BINARY:-}" NO_CACHE="${NO_CACHE:-false}" S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT="MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" +S3TESTS_ENABLE_LOCAL_KMS="${S3TESTS_ENABLE_LOCAL_KMS:-true}" +S3_KMS_KEY_ID="${S3_KMS_KEY_ID:-rustfs-s3tests-default-key}" +S3_KMS_SECONDARY_KEY_ID="${S3_KMS_SECONDARY_KEY_ID:-rustfs-s3tests-secondary-key}" +if [ "${S3TESTS_ENABLE_LOCAL_KMS}" = "true" ] && [ "${DEPLOY_MODE}" != "existing" ]; then + # An explicitly empty value exercises the missing-default-key path. + S3_KMS_DEFAULT_KEY_ID="${S3_KMS_DEFAULT_KEY_ID-${S3_KMS_KEY_ID}}" +else + S3_KMS_DEFAULT_KEY_ID="${S3_KMS_DEFAULT_KEY_ID:-}" +fi # Additional directories (SCRIPT_DIR and PROJECT_ROOT defined earlier) ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts/s3tests-${TEST_MODE}" @@ -252,6 +261,9 @@ else fi DATA_DIR="${DATA_BASE}/test-data/${CONTAINER_NAME}" RUSTFS_PID="" +RUSTFS_KMS_ARGS=() +S3TESTS_KMS_HOST_KEY_DIR="${S3TESTS_KMS_KEY_DIR:-${DATA_BASE}/kms-keys/${CONTAINER_NAME}}" +S3TESTS_KMS_RUNTIME_KEY_DIR="${S3TESTS_KMS_HOST_KEY_DIR}" if [ "${DEPLOY_MODE}" != "existing" ] && [ -z "${RUSTFS_SSE_S3_MASTER_KEY:-}" ]; then export RUSTFS_SSE_S3_MASTER_KEY="${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}" @@ -282,6 +294,11 @@ Environment Variables: S3_ALT_ACCESS_KEY - Alt user access key (default: rustfsalt) S3_ALT_SECRET_KEY - Alt user secret key (default: rustfsalt) RUSTFS_SSE_S3_MASTER_KEY - Optional base64 32-byte key for local managed SSE fallback + S3TESTS_ENABLE_LOCAL_KMS - Enable local KMS for SSE-KMS cases (default: true) + S3_KMS_KEY_ID - s3-tests KMS key id (default: rustfs-s3tests-default-key) + S3_KMS_SECONDARY_KEY_ID - Second KMS key id (default: rustfs-s3tests-secondary-key) + S3_KMS_DEFAULT_KEY_ID - Expected default key (managed local KMS: primary key; existing: none; empty disables it) + S3TESTS_KMS_KEY_DIR - Host key directory for local KMS (default: DATA_ROOT/kms-keys) RUSTFS_SCANNER_ENABLED - Enable background scanner for harness service (default: false) MAXFAIL - Stop after N failures, 0 = never stop (default: 1) XDIST - Enable parallel execution with N workers (default: 0) @@ -345,6 +362,68 @@ cleanup() { trap cleanup EXIT +prepare_s3tests_local_kms() { + if [ "${S3TESTS_ENABLE_LOCAL_KMS}" != "true" ]; then + return 0 + fi + if [ "${DEPLOY_MODE}" = "existing" ]; then + log_warn "Skipping local KMS setup for DEPLOY_MODE=existing; configure both test key IDs and the expected default for the target service" + return 0 + fi + if [ "${S3_KMS_KEY_ID}" = "${S3_KMS_SECONDARY_KEY_ID}" ]; then + log_error "S3_KMS_KEY_ID and S3_KMS_SECONDARY_KEY_ID must differ for cross-key tests" + return 1 + fi + if [ "${DEPLOY_MODE}" = "docker" ] && [ -z "${S3TESTS_KMS_KEY_DIR:-}" ]; then + S3TESTS_KMS_HOST_KEY_DIR="/tmp/${CONTAINER_NAME}/kms-keys" + S3TESTS_KMS_RUNTIME_KEY_DIR="/data/kms-keys" + fi + + mkdir -p "${S3TESTS_KMS_HOST_KEY_DIR}" + chmod 700 "${S3TESTS_KMS_HOST_KEY_DIR}" + local key_id key_material + for key_id in "${S3_KMS_KEY_ID}" "${S3_KMS_SECONDARY_KEY_ID}"; do + # Public, distinct test fixtures stay stable when DATA_ROOT is reused. + key_material="${S3TESTS_LOCAL_SSE_MASTER_KEY_DEFAULT}" + if [ "${key_id}" = "${S3_KMS_SECONDARY_KEY_ID}" ]; then + key_material=$(python3 -c 'import base64, hashlib; print(base64.b64encode(hashlib.sha256(b"rustfs-s3-tests-secondary-fixture").digest()).decode())') + fi + cat > "${S3TESTS_KMS_HOST_KEY_DIR}/${key_id}.key" < "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log" 2>&1 & @@ -472,6 +554,7 @@ elif [ "${DEPLOY_MODE}" = "build" ]; then --address "${S3_HOST}:${S3_PORT}" \ --access-key "${S3_ACCESS_KEY}" \ --secret-key "${S3_SECRET_KEY}" \ + "${RUSTFS_KMS_ARGS[@]}" \ "${DATA_DIR}/rustfs0" "${DATA_DIR}/rustfs1" "${DATA_DIR}/rustfs2" "${DATA_DIR}/rustfs3" \ > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log" 2>&1 & @@ -506,6 +589,11 @@ elif [ "${DEPLOY_MODE}" = "docker" ]; then -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \ -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \ -e RUSTFS_SSE_S3_MASTER_KEY="${RUSTFS_SSE_S3_MASTER_KEY}" \ + -e RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS="${RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS:-false}" \ + -e RUSTFS_KMS_ENABLE="${RUSTFS_KMS_ENABLE:-false}" \ + -e RUSTFS_KMS_BACKEND="${RUSTFS_KMS_BACKEND:-local}" \ + -e RUSTFS_KMS_KEY_DIR="${RUSTFS_KMS_KEY_DIR:-}" \ + -e RUSTFS_KMS_DEFAULT_KEY_ID \ -e RUSTFS_SCANNER_ENABLED="${RUSTFS_SCANNER_ENABLED}" \ -e RUSTFS_SCANNER_START_DELAY_SECS="${RUSTFS_SCANNER_START_DELAY_SECS}" \ -e RUSTFS_SCANNER_CYCLE="${RUSTFS_SCANNER_CYCLE}" \ @@ -761,6 +849,15 @@ envsubst < "${TEMPLATE_PATH}" > "${CONF_OUTPUT_PATH}" || { log_error "Failed to generate s3tests config" exit 1 } +if [ -n "${S3_KMS_KEY_ID:-}" ]; then + tmp_conf="${CONF_OUTPUT_PATH}.tmp" + sed \ + -e "s|^#kms_keyid = .*$|kms_keyid = ${S3_KMS_KEY_ID}|" \ + -e "s|^#kms_keyid2 = .*$|kms_keyid2 = ${S3_KMS_SECONDARY_KEY_ID}|" \ + -e "s|^#kms_default_keyid =.*$|kms_default_keyid = ${S3_KMS_DEFAULT_KEY_ID}|" \ + "${CONF_OUTPUT_PATH}" > "${tmp_conf}" + mv "${tmp_conf}" "${CONF_OUTPUT_PATH}" +fi # Step 7: Provision s3-tests alt user # Note: Main user (rustfsadmin) is a system user and doesn't need to be created via API diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py index a76643b3a..a8049c16a 100644 --- a/scripts/scanner_abba.py +++ b/scripts/scanner_abba.py @@ -14,21 +14,41 @@ import signal import subprocess import sys import time +from datetime import datetime, timezone SCENARIOS = ("cold-hot", "fresh-hot", "multi-hot-new", "running-heal", "mrf-replay") LEGS = ("A1", "B1", "B2", "A2") MAX_JSON_BYTES = 1024 * 1024 METRICS = ( - "p99_ms", "throughput_ops", "rss_bytes", "cpu_seconds", "iops", "rpc_count", + "p95_ms", "p99_ms", "throughput_ops", "rss_bytes", "cpu_seconds", "iops", "rpc_count", "cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds", "walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests", "foreground_pressure_samples", "foreground_pressure_high_samples", "heal_mainline_throttle_delayed", "heal_lock_wait_p99_ms", "heal_attempts", "heal_attempt_failures", "heal_retry_attempts", + "heal_start_p95_ms", "heal_duplicate_task_count", "heal_lock_hold_p95_ms", ) REPEATABILITY_LIMIT = Decimal("0.05") P2_WORK_MULTIPLE_LIMIT = Decimal("1.2") +W11_RSS_GROWTH_LIMIT = Decimal("0.05") +RELEASE_PROFILE_ARTIFACTS = ( + "allocation-profile", + "flamegraph", + "rss-samples", + "save-frequency", +) +MIN_MEASURED_RELEASE_DURATION_SECONDS = 7200 +RELEASE_FAULT_MODES = ( + "process-restart", + "process-crash-restart", +) +RELEASE_SCHEDULER_BOUNDS = ( + "admission-retry-idempotency", + "deadline-budget", + "lock-hold-bound", + "minimum-progress", +) def require(condition, message): @@ -120,7 +140,7 @@ def validate_manifest(manifest): number(fixed.get("offered_load_ops"), "offered load", 1) require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10, "rounds must be 3..10") - minimum = 900 if manifest["evidence"] == "measured" else 1 + minimum = MIN_MEASURED_RELEASE_DURATION_SECONDS if manifest["evidence"] == "measured" else 1 require(type(manifest.get("duration_seconds")) is int and minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds") number(manifest.get("min_free_bytes"), "min_free_bytes", 1) @@ -140,6 +160,116 @@ def validate_manifest(manifest): number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs") if scenario in ("running-heal", "mrf-replay"): require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs") + validate_release_evidence_manifest(manifest) + + +def release_evidence_integer(value, name, minimum=1, maximum=1024): + require(type(value) is int and minimum <= value <= maximum, f"invalid release_evidence.{name}") + return value + + +def release_evidence_string(value, name): + require(isinstance(value, str) and value.strip(), f"missing release_evidence.{name}") + return value + + +def release_evidence_bool(value, name): + require(type(value) is bool, f"invalid release_evidence.{name}") + return value + + +def release_evidence_true(value, name): + release_evidence_bool(value, name) + require(value is True, f"missing release_evidence.{name}") + + +def release_evidence_exact_strings(value, expected, name): + require(isinstance(value, list) and all(isinstance(item, str) and item.strip() for item in value), + f"invalid release_evidence.{name}") + observed = set(value) + require(len(observed) == len(value), f"duplicate release_evidence.{name}") + missing = sorted(set(expected) - observed) + require(not missing, f"missing release_evidence.{name}: {', '.join(missing)}") + unknown = sorted(observed - set(expected)) + require(not unknown, f"unknown release_evidence.{name}: {', '.join(unknown)}") + return value + + +def validate_release_evidence_manifest(manifest): + if manifest["evidence"] != "measured": + return + + evidence = manifest.get("release_evidence") + require(isinstance(evidence, dict), "missing release_evidence for measured ABBA") + + topology = evidence.get("topology") + require(isinstance(topology, dict), "missing release_evidence.topology") + nodes = release_evidence_integer(topology.get("nodes"), "topology.nodes", 3, 64) + drives = release_evidence_integer(topology.get("drives_per_node"), "topology.drives_per_node", 1, 64) + set_size = release_evidence_integer(topology.get("erasure_set_size"), "topology.erasure_set_size", 12, 12) + data = release_evidence_integer(topology.get("erasure_data_blocks"), "topology.erasure_data_blocks", 8, 8) + parity = release_evidence_integer(topology.get("erasure_parity_blocks"), "topology.erasure_parity_blocks", 4, 4) + require(data + parity == set_size, "release_evidence.topology must be EC8+4") + require(nodes * drives >= set_size, "release_evidence.topology cannot host one EC8+4 set") + pools = release_evidence_integer(topology.get("pools"), "topology.pools", 1) + sets_total = release_evidence_integer(topology.get("sets_total"), "topology.sets_total", 1) + sampled_pools = release_evidence_integer(topology.get("sampled_pools"), "topology.sampled_pools", 2) + sampled_sets = release_evidence_integer(topology.get("sampled_sets"), "topology.sampled_sets", 2) + require(sampled_pools <= pools, "release_evidence.topology sampled pools exceed total pools") + require(sampled_sets <= sets_total, "release_evidence.topology sampled sets exceed total sets") + + distributed = evidence.get("distributed") + require(isinstance(distributed, dict), "missing release_evidence.distributed") + endpoints = distributed.get("metrics_endpoints") + require(isinstance(endpoints, list) and len(endpoints) >= nodes, "missing release_evidence.distributed.metrics_endpoints") + require( + all(isinstance(endpoint, str) and endpoint.strip() for endpoint in endpoints) + and len(set(endpoints)) == len(endpoints), + "invalid release_evidence.distributed.metrics_endpoints", + ) + release_evidence_string(distributed.get("failure_domain"), "distributed.failure_domain") + release_evidence_true(distributed.get("same_window_sampling"), "distributed.same_window_sampling") + + scheduler = evidence.get("scheduler") + require(isinstance(scheduler, dict), "missing release_evidence.scheduler") + release_evidence_exact_strings(scheduler.get("bounds"), RELEASE_SCHEDULER_BOUNDS, "scheduler.bounds") + release_evidence_integer(scheduler.get("max_deferred_items"), "scheduler.max_deferred_items", 1, 2**31 - 1) + release_evidence_integer(scheduler.get("max_deferred_bytes"), "scheduler.max_deferred_bytes", 1, 2**63 - 1) + release_evidence_integer(scheduler.get("max_retry_age_seconds"), "scheduler.max_retry_age_seconds", 1, 86400) + release_evidence_true(scheduler.get("duplicate_task_bound_observed"), "scheduler.duplicate_task_bound_observed") + + crash = evidence.get("crash_restart") + require(isinstance(crash, dict), "missing release_evidence.crash_restart") + release_evidence_exact_strings(crash.get("fault_modes"), RELEASE_FAULT_MODES, "crash_restart.fault_modes") + release_evidence_true(crash.get("unclean_shutdown_marker"), "crash_restart.unclean_shutdown_marker") + + mixed = evidence.get("mixed_version") + require(isinstance(mixed, dict), "missing release_evidence.mixed_version") + baseline_revision = manifest["baseline"]["revision"] + candidate_revision = manifest["candidate"]["revision"] + require(baseline_revision != candidate_revision, + "release_evidence.mixed_version requires distinct baseline and candidate revisions") + require(manifest["baseline"]["sha256"] != manifest["candidate"]["sha256"], + "release_evidence.mixed_version requires distinct baseline and candidate binaries") + revisions = mixed.get("participating_revisions") + require( + isinstance(revisions, list) + and len(set(revisions)) >= 2 + and all(isinstance(revision, str) and len(revision) == 40 and all(c in "0123456789abcdef" for c in revision) + for revision in revisions), + "invalid release_evidence.mixed_version.participating_revisions", + ) + for revision in (baseline_revision, candidate_revision): + require(revision in revisions, "release_evidence.mixed_version omits tested build revision") + for key in ("reader", "writer", "rollback_payload"): + require(mixed.get(key) is True, f"missing release_evidence.mixed_version.{key}") + + profile = evidence.get("profile") + require(isinstance(profile, dict), "missing release_evidence.profile") + release_evidence_exact_strings(profile.get("required_artifacts"), RELEASE_PROFILE_ARTIFACTS, + "profile.required_artifacts") + for key in ("collector_config_sha256", "profiler_config_sha256"): + require(sha(profile.get(key)), f"invalid release_evidence.profile.{key}") class OwnedCommand: @@ -254,6 +384,8 @@ def validate_result(result, request, expected): require(result.get("build") == request["build"], "deployed build provenance mismatch") require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch") require(result.get("background") == request["background"], "background mode mismatch") + if request["evidence"] == "measured": + require(result.get("release_evidence") == request["release_evidence"], "release evidence provenance mismatch") require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600, "sample_count must be 1..3600") number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"]) @@ -267,6 +399,9 @@ def validate_result(result, request, expected): "foreground pressure high samples exceed samples") require(metrics["heal_attempt_failures"] <= metrics["heal_attempts"], "heal failures exceed attempts") require(metrics["heal_retry_attempts"] <= metrics["heal_attempts"], "heal retries exceed attempts") + require(metrics["heal_duplicate_task_count"] == 0, "duplicate heal task admission") + require(metrics["heal_start_p95_ms"] > 0, "zero heal start p95") + require(metrics["heal_lock_hold_p95_ms"] > 0, "zero heal lock hold p95") require(metrics["errors"] == 0, "workload request errors") require(metrics["cold_walk_objects"] <= metrics["walk_objects"], "cold walk exceeds total walk") require(result.get("oracle") == expected, "object/version/byte oracle mismatch") @@ -346,6 +481,48 @@ def running_heal_pacing(group, baseline, candidate, p99, throughput, noisy): } +def bounded_retry_window(group, baseline, candidate, p99, throughput, noisy, candidate_attempt_costs): + if group[0]["scenario"] != "running-heal" or group[0]["comparison"] != "build": + return {"status": "not_applicable"} + + rss_growth = relative_change_or_none(candidate["rss_bytes"], baseline["rss_bytes"], "rss_bytes") + lock_wait_change = relative_change_or_none( + candidate["heal_lock_wait_p99_ms"], baseline["heal_lock_wait_p99_ms"], "heal lock wait p99", + ) + latency_improved = p99 < 0 or throughput > 0 + lock_wait_improved = lock_wait_change is not None and lock_wait_change < 0 + rss_within_limit = rss_growth is not None and rss_growth <= W11_RSS_GROWTH_LIMIT + attempt_cost_available = bool(candidate_attempt_costs) + status = ( + "inconclusive" + if noisy + else "observed" + if latency_improved and lock_wait_improved and rss_within_limit and attempt_cost_available + else "rss_regression" + if latency_improved and lock_wait_improved and not rss_within_limit + else "no_measured_benefit" + if attempt_cost_available + else "pending" + ) + return { + "status": status, + "rss_growth_limit": float(W11_RSS_GROWTH_LIMIT), + "rss_growth": None if rss_growth is None else float(rss_growth), + "rss_within_limit": rss_within_limit, + "baseline_rss_bytes": float(baseline["rss_bytes"]), + "candidate_rss_bytes": float(candidate["rss_bytes"]), + "baseline_heal_lock_wait_p99_ms": float(baseline["heal_lock_wait_p99_ms"]), + "candidate_heal_lock_wait_p99_ms": float(candidate["heal_lock_wait_p99_ms"]), + "heal_lock_wait_p99_change": None if lock_wait_change is None else float(lock_wait_change), + "healthy_page_latency_observed": latency_improved, + "foreground_p99_change": float(p99), + "foreground_throughput_change": float(throughput), + "candidate_attempt_cost_per_healed_object": ( + None if not candidate_attempt_costs else float(max(candidate_attempt_costs)) + ), + } + + def convergence(result): window = result.get("convergence") if not window or window.get("writes_stopped") is not True or window.get("last_mutation_observed") is not True or window.get("first_complete_publication") is not True: @@ -376,6 +553,7 @@ def evaluate(cells): noise = max(drift, repeat_drift) > REPEATABILITY_LIMIT a = {key: (decimal_number(a1[key], key) + decimal_number(a2[key], key)) / Decimal("2") for key in METRICS} b = {key: (decimal_number(b1[key], key) + decimal_number(b2[key], key)) / Decimal("2") for key in METRICS} + p95 = max(a["p95_ms"], b["p95_ms"]) p99 = relative_change(b["p99_ms"], a["p99_ms"], "p99_ms") throughput = relative_change(b["throughput_ops"], a["throughput_ops"], "throughput_ops") thresholds = {"p99_regression": Decimal("0.10") if control else Decimal("0.05"), @@ -392,7 +570,11 @@ def evaluate(cells): required = ratio(a["cold_walk_objects"], a["walk_objects"], "cold walk baseline") * Decimal("0.80") reduction = Decimal("1") - ratio(b["walk_objects"], a["walk_objects"], "walk reduction") p1 = {"required_reduction": float(required), "observed_reduction": float(reduction), - "repeatability_drift": report_number(work_drift)} + "repeatability_drift": report_number(work_drift), + "baseline_walk_objects": int(a["walk_objects"]), + "baseline_cold_walk_objects": int(a["cold_walk_objects"]), + "candidate_walk_objects": int(b["walk_objects"]), + "candidate_cold_walk_objects": int(b["cold_walk_objects"])} if group[0]["scenario"] == "cold-hot": # Compare counts before division can round repeating decimal ratios. passed &= a["walk_objects"] - b["walk_objects"] >= a["cold_walk_objects"] * Decimal("0.80") @@ -406,12 +588,17 @@ def evaluate(cells): value for cell, value in zip(group, attempt_costs) if cell["leg"].startswith("B") and value is not None ] w10 = running_heal_pacing(group, a, b, p99, throughput, noise) + w11 = bounded_retry_window(group, a, b, p99, throughput, noise, candidate_attempt_costs) inconclusive |= noise or p2_pending if not noise and not passed: failed = True comparisons.append({"scenario": group[0]["scenario"], "comparison": group[0]["comparison"], "round": group[0]["round"], "status": "inconclusive" if noise else ("fail" if not passed else "inconclusive" if p2_pending else "pass"), "a2_a1_drift": report_number(drift), "b2_b1_drift": report_number(repeat_drift), + "foreground_p95_ms": float(p95), + "foreground_p99_ms": float(max(a["p99_ms"], b["p99_ms"])), + "throughput_ops": float(min(a["throughput_ops"], b["throughput_ops"])), + "error_rate": float(max(a["errors"] / a["requests"], b["errors"] / b["requests"])), "p99_regression": float(p99), "throughput_change": float(throughput), "thresholds": {key: float(value) for key, value in thresholds.items()}, "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT), @@ -422,7 +609,14 @@ def evaluate(cells): "candidate_vs_baseline": scanner_cache_cost_change(b, a), }, "w10": w10, + "w11": w11, "w10_w11": { + "foreground_pressure_samples": [ + cell["result"]["metrics"]["foreground_pressure_samples"] for cell in group + ], + "foreground_pressure_high_samples": [ + cell["result"]["metrics"]["foreground_pressure_high_samples"] for cell in group + ], "foreground_pressure_high_sample_ratios": [ float(pressure_high_ratio(cell["result"]["metrics"])) for cell in group ], @@ -435,6 +629,17 @@ def evaluate(cells): "candidate_attempt_cost_per_healed_object": ( None if not candidate_attempt_costs else float(max(candidate_attempt_costs)) ), + }, + "w09": { + "heal_start_p95_ms": [ + cell["result"]["metrics"]["heal_start_p95_ms"] for cell in group + ], + "heal_duplicate_task_count": [ + cell["result"]["metrics"]["heal_duplicate_task_count"] for cell in group + ], + "heal_lock_hold_p95_ms": [ + cell["result"]["metrics"]["heal_lock_hold_p95_ms"] for cell in group + ], }}) return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons @@ -445,6 +650,9 @@ def collect_live(prepared, request, request_path, adapter): connection = prepared["collector"] require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection") require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint") + expected_metrics_endpoints = None + if request.get("evidence") == "measured": + expected_metrics_endpoints = request["release_evidence"]["distributed"]["metrics_endpoints"] output = request_path.parent / "telemetry" args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"], "--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed", @@ -470,6 +678,9 @@ def collect_live(prepared, request, request_path, adapter): require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response") metrics = list((output / "metrics").glob("admin-metrics.*.ndjson")) endpoints = [endpoint for endpoint in connection["metrics_endpoints"].split(",") if endpoint] + if expected_metrics_endpoints is not None: + require(endpoints == expected_metrics_endpoints, + "collector metrics endpoints do not match release evidence") require(metrics and len(metrics) == len(endpoints) * len(samples), "missing distributed metrics samples") for sample in metrics: # The collector requests n=1, so each file contains one final JSON record. @@ -497,6 +708,8 @@ def run(manifest, adapter, output, data_root): require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space") manifest["adapter_sha256"] = digest(adapter) manifest["collector_sha256"] = digest(Path(__file__).with_name("run_scanner_validation_harness.sh")) + started_at = datetime.now(timezone.utc).replace(microsecond=0) + manifest["started_at"] = started_at.isoformat().replace("+00:00", "Z") write_json(output / "manifest.json", manifest) cells = [] write_json(output / "report.json", {"status": "incomplete", "performance": "pending"}) @@ -518,6 +731,8 @@ def run(manifest, adapter, output, data_root): "duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir), "expected_healed_objects": manifest["expected_healed_objects"][scenario], "expected_oracle": manifest["oracles"][scenario]} + if manifest["evidence"] == "measured": + request["release_evidence"] = manifest["release_evidence"] require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run") require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run") require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space") @@ -543,14 +758,19 @@ def run(manifest, adapter, output, data_root): require(stopped.get("stopped") is True, "adapter failed to stop deployment") status, comparisons = evaluate(cells) synthetic = manifest["evidence"] == "synthetic" + finished_at = datetime.now(timezone.utc).replace(microsecond=0) report = {"status": "synthetic_validated" if synthetic and status == "pass" else status, "evidence": manifest["evidence"], "performance": "pending" if synthetic else status, - "cells": len(cells), "comparisons": comparisons} + "cells": len(cells), "comparisons": comparisons, + "started_at": manifest["started_at"], "finished_at": finished_at.isoformat().replace("+00:00", "Z")} write_json(output / "report.json", report) return 0 if status == "pass" else 3 if status == "inconclusive" else 1 except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error: + finished_at = datetime.now(timezone.utc).replace(microsecond=0) write_json(output / "report.json", {"status": "failed", "performance": "pending", - "completed_cells": len(cells), "error": str(error)}) + "completed_cells": len(cells), "error": str(error), + "started_at": manifest.get("started_at"), + "finished_at": finished_at.isoformat().replace("+00:00", "Z")}) raise diff --git a/scripts/summarize_scanner_heal_perf.py b/scripts/summarize_scanner_heal_perf.py index 36f5da782..7b7751ce9 100755 --- a/scripts/summarize_scanner_heal_perf.py +++ b/scripts/summarize_scanner_heal_perf.py @@ -5,17 +5,29 @@ from __future__ import annotations import argparse from collections import Counter +from datetime import datetime, timezone from decimal import Decimal import hashlib import json from pathlib import Path +import shutil import sys from typing import Any +from scanner_abba import ( + LEGS, + MIN_MEASURED_RELEASE_DURATION_SECONDS, + RELEASE_PROFILE_ARTIFACTS, + RELEASE_SCHEDULER_BOUNDS, + SCENARIOS, + validate_release_evidence_manifest, +) + MAX_JSON_BYTES = 1024 * 1024 CACHE_COST_PREFIX = "CACHE_COST " PASS_STATES = {"pass"} FAIL_STATES = {"fail", "failed"} +RELEASE_DESCRIPTOR_GATES = ("G10", "P1", "P3") def require(condition: bool, message: str) -> None: @@ -77,12 +89,154 @@ def max_decimal(values: list[Decimal | None]) -> Decimal | None: return max(present) +def require_integer(value: Any, name: str, minimum: int = 0) -> int: + require(type(value) is int and value >= minimum, f"invalid integer field: {name}") + return value + + +def timestamp(value: Any, name: str) -> str: + require(isinstance(value, str) and value.strip(), f"missing timestamp: {name}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + require(parsed.tzinfo is not None, f"timestamp must include timezone: {name}") + return parsed.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def require_metric_series(value: Any, name: str, minimum: Decimal | None = None, + maximum: Decimal | None = None) -> list[Decimal | None]: + require(isinstance(value, list) and value, f"missing performance evidence field: {name}") + parsed = [maybe_number(item, name) for item in value] + for item in parsed: + if item is None: + continue + if minimum is not None: + require(item >= minimum, f"{name} below minimum") + if maximum is not None: + require(item <= maximum, f"{name} above maximum") + return parsed + + +def require_measured_comparison_evidence(comparison: dict[str, Any], index: int) -> None: + w10_w11 = comparison.get("w10_w11") + require(isinstance(w10_w11, dict), f"comparison {index} missing W10/W11 evidence") + pressure = require_metric_series( + w10_w11.get("foreground_pressure_high_sample_ratios"), + f"comparison {index} foreground_pressure_high_sample_ratios", + Decimal("0"), + Decimal("1"), + ) + lock_wait = require_metric_series( + w10_w11.get("heal_lock_wait_p99_ms"), + f"comparison {index} heal_lock_wait_p99_ms", + Decimal("0"), + ) + attempt_cost = require_metric_series( + w10_w11.get("attempt_cost_per_healed_object"), + f"comparison {index} attempt_cost_per_healed_object", + Decimal("0"), + ) + require(len(pressure) == len(lock_wait) == len(attempt_cost), + f"comparison {index} W10/W11 evidence length mismatch") + candidate_attempt_cost = maybe_number( + w10_w11.get("candidate_attempt_cost_per_healed_object"), + f"comparison {index} candidate_attempt_cost_per_healed_object", + ) + require(candidate_attempt_cost is None or candidate_attempt_cost >= 0, + f"comparison {index} candidate attempt cost below minimum") + w09 = comparison.get("w09") + require(isinstance(w09, dict), f"comparison {index} missing W09 evidence") + start_p95 = require_metric_series( + w09.get("heal_start_p95_ms"), + f"comparison {index} heal_start_p95_ms", + Decimal("0"), + ) + require(all(item is not None and item > 0 for item in start_p95), + f"comparison {index} heal_start_p95_ms must be measured") + duplicate_tasks = require_metric_series( + w09.get("heal_duplicate_task_count"), + f"comparison {index} heal_duplicate_task_count", + Decimal("0"), + Decimal("0"), + ) + require(all(item is not None and item == 0 for item in duplicate_tasks), + f"comparison {index} heal_duplicate_task_count must be measured") + lock_hold = require_metric_series( + w09.get("heal_lock_hold_p95_ms"), + f"comparison {index} heal_lock_hold_p95_ms", + Decimal("0"), + ) + require(all(item is not None and item > 0 for item in lock_hold), + f"comparison {index} heal_lock_hold_p95_ms must be measured") + require(len(start_p95) == len(duplicate_tasks) == len(lock_hold), + f"comparison {index} W09 evidence length mismatch") + w11 = comparison.get("w11") + require(isinstance(w11, dict), f"comparison {index} missing W11 evidence") + status = w11.get("status") + require(status in {"observed", "no_measured_benefit", "rss_regression", "pending", "inconclusive", "not_applicable"}, + f"comparison {index} invalid W11 evidence status") + if comparison.get("scenario") == "running-heal" and comparison.get("comparison") == "build": + require(status == "observed", f"comparison {index} W11 bounded retry evidence was not observed") + for key in ( + "rss_growth_limit", + "rss_growth", + "baseline_rss_bytes", + "candidate_rss_bytes", + "baseline_heal_lock_wait_p99_ms", + "candidate_heal_lock_wait_p99_ms", + "heal_lock_wait_p99_change", + "foreground_p99_change", + "foreground_throughput_change", + "candidate_attempt_cost_per_healed_object", + ): + value = maybe_number(w11.get(key), f"comparison {index} W11 {key}") + require(value is not None, f"comparison {index} W11 {key} is required") + require(w11.get("rss_within_limit") is True, f"comparison {index} W11 RSS growth is outside limit") + require(w11.get("healthy_page_latency_observed") is True, + f"comparison {index} W11 healthy-page latency benefit is required") + + +def require_complete_abba_matrix(manifest: dict[str, Any], report: dict[str, Any], comparisons: list[dict[str, Any]]) -> None: + require(report.get("evidence") == manifest.get("evidence"), "manifest/report evidence mismatch") + require(type(manifest.get("duration_seconds")) is int and + manifest["duration_seconds"] >= MIN_MEASURED_RELEASE_DURATION_SECONDS, + "measured ABBA duration_seconds requires at least two hours") + rounds = manifest.get("rounds") + require(type(rounds) is int and 3 <= rounds <= 10, "invalid manifest.rounds") + expected_cells = len(SCENARIOS) * 2 * rounds * len(LEGS) + require( + report.get("cells") == expected_cells, + f"ABBA matrix cell count mismatch: expected {expected_cells}, got {report.get('cells')}", + ) + expected_keys = { + (scenario, comparison, round_id) + for scenario in SCENARIOS + for comparison in ("build", "background") + for round_id in range(1, rounds + 1) + } + observed_keys = [] + for index, comparison in enumerate(comparisons): + key = (comparison.get("scenario"), comparison.get("comparison"), comparison.get("round")) + require(key in expected_keys, f"comparison {index} is outside the ABBA matrix") + require(comparison.get("status") in PASS_STATES, f"comparison {index} did not pass") + observed_keys.append(key) + observed_set = set(observed_keys) + require(len(observed_keys) == len(observed_set), "duplicate ABBA matrix comparison") + missing = sorted(expected_keys - observed_set) + require(not missing, f"missing ABBA matrix comparison: {missing[0] if missing else ''}") + + def summarize_abba(abba_dir: Path) -> dict[str, Any]: manifest_path = abba_dir / "manifest.json" report_path = abba_dir / "report.json" manifest = read_json(manifest_path) report = read_json(report_path) + report_state = report.get("status") + performance_state = report.get("performance") + require(isinstance(report_state, str) and report_state, "report.status missing") + require(isinstance(performance_state, str) and performance_state, "report.performance missing") comparisons = report.get("comparisons") + if comparisons is None: + require(report_state not in PASS_STATES, "passing report requires comparisons") + comparisons = [] require(isinstance(comparisons, list), "report.comparisons must be a list") counts = Counter() @@ -90,6 +244,18 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: throughput_losses: list[Decimal] = [] p1_rows = [] p2_values: list[Decimal | None] = [] + foreground_p95_values: list[Decimal | None] = [] + foreground_p99_values: list[Decimal | None] = [] + throughput_values: list[Decimal | None] = [] + error_rate_values: list[Decimal | None] = [] + pressure_samples = 0 + pressure_high_samples = 0 + attempt_cost_samples = 0 + start_p95_values: list[Decimal | None] = [] + duplicate_task_values: list[Decimal | None] = [] + lock_hold_values: list[Decimal | None] = [] + w10_rows = [] + w11_rows = [] for index, comparison in enumerate(comparisons): require(isinstance(comparison, dict), f"comparison {index} must be an object") state = comparison.get("status") @@ -98,6 +264,8 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: p99_regressions.append(number(comparison.get("p99_regression"), f"comparison {index} p99_regression")) throughput_change = number(comparison.get("throughput_change"), f"comparison {index} throughput_change") throughput_losses.append(max(Decimal("0"), -throughput_change)) + foreground_p95_values.append(maybe_number(comparison.get("foreground_p95_ms"), "foreground_p95_ms")) + foreground_p99_values.append(maybe_number(comparison.get("foreground_p99_ms"), "foreground_p99_ms")) p1 = comparison.get("p1") if isinstance(p1, dict): p1_rows.append({ @@ -110,20 +278,71 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: None if p1.get("repeatability_drift") is None else float(number(p1.get("repeatability_drift"), "p1.repeatability_drift")) ), + "baseline_walk_objects": p1.get("baseline_walk_objects"), + "baseline_cold_walk_objects": p1.get("baseline_cold_walk_objects"), + "candidate_walk_objects": p1.get("candidate_walk_objects"), + "candidate_cold_walk_objects": p1.get("candidate_cold_walk_objects"), }) p2 = comparison.get("p2_post_stop_work_multiples") if isinstance(p2, list): p2_values.extend(maybe_number(value, "p2_post_stop_work_multiple") for value in p2) + w09 = comparison.get("w09") + if isinstance(w09, dict): + for value in w09.get("heal_start_p95_ms", []): + start_p95_values.append(maybe_number(value, "heal_start_p95_ms")) + for value in w09.get("heal_duplicate_task_count", []): + duplicate_task_values.append(maybe_number(value, "heal_duplicate_task_count")) + for value in w09.get("heal_lock_hold_p95_ms", []): + lock_hold_values.append(maybe_number(value, "heal_lock_hold_p95_ms")) + w10_w11 = comparison.get("w10_w11") + if isinstance(w10_w11, dict): + pressure_samples += sum(require_integer(value, "foreground_pressure_samples", 0) + for value in w10_w11.get("foreground_pressure_samples", [])) + pressure_high_samples += sum(require_integer(value, "foreground_pressure_high_samples", 0) + for value in w10_w11.get("foreground_pressure_high_samples", [])) + for value in w10_w11.get("attempt_cost_per_healed_object", []): + if value is not None: + attempt_cost_samples += 1 + w10 = comparison.get("w10") + if isinstance(w10, dict) and comparison.get("scenario") == "running-heal" and comparison.get("comparison") == "build": + w10_rows.append({ + "round": comparison.get("round"), + "status": w10.get("status"), + "pacing_observed": w10.get("pacing_observed"), + "candidate_pressure_high_ratio": w10.get("candidate_pressure_high_ratio"), + "candidate_delay_events": w10.get("candidate_delay_events"), + "foreground_p99_change": w10.get("foreground_p99_change"), + "foreground_throughput_change": w10.get("foreground_throughput_change"), + }) + if "throughput_ops" in comparison: + throughput_values.append(maybe_number(comparison.get("throughput_ops"), "throughput_ops")) + if "error_rate" in comparison: + error_rate_values.append(maybe_number(comparison.get("error_rate"), "error_rate")) + w11 = comparison.get("w11") + if isinstance(w11, dict) and comparison.get("scenario") == "running-heal" and comparison.get("comparison") == "build": + w11_rows.append({ + "round": comparison.get("round"), + "status": w11.get("status"), + "rss_growth": w11.get("rss_growth"), + "rss_growth_limit": w11.get("rss_growth_limit"), + "heal_lock_wait_p99_change": w11.get("heal_lock_wait_p99_change"), + "foreground_p99_change": w11.get("foreground_p99_change"), + "foreground_throughput_change": w11.get("foreground_throughput_change"), + "candidate_attempt_cost_per_healed_object": w11.get("candidate_attempt_cost_per_healed_object"), + }) - report_state = report.get("status") - performance_state = report.get("performance") - require(isinstance(report_state, str) and report_state, "report.status missing") - require(isinstance(performance_state, str) and performance_state, "report.performance missing") measured = report.get("evidence") == "measured" passed = report_state in PASS_STATES and performance_state in PASS_STATES and measured + if passed: + require_complete_abba_matrix(manifest, report, comparisons) + validate_release_evidence_manifest({**manifest, "evidence": "measured"}) + for index, comparison in enumerate(comparisons): + require_measured_comparison_evidence(comparison, index) gate_state = "pass" if passed else "fail" if report_state == "synthetic_validated": reason = "synthetic evidence validates the harness only; measured performance remains pending" + elif report_state in FAIL_STATES and isinstance(report.get("error"), str) and report["error"]: + reason = f"ABBA report status is {report_state}: {report['error']}" elif report_state not in PASS_STATES: reason = f"ABBA report status is {report_state}" elif performance_state not in PASS_STATES: @@ -142,12 +361,26 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: "performance": performance_state, "evidence": report.get("evidence"), "cells": report.get("cells", 0), + "completed_cells": report.get("completed_cells"), + "error": report.get("error"), "comparisons_total": len(comparisons), "comparison_status_counts": dict(sorted(counts.items())), "worst_p99_regression": None if not p99_regressions else float(max(p99_regressions)), "worst_throughput_loss": None if not throughput_losses else float(max(throughput_losses)), + "foreground_p95_ms": None if max_decimal(foreground_p95_values) is None else float(max_decimal(foreground_p95_values)), + "foreground_p99_ms": None if max_decimal(foreground_p99_values) is None else float(max_decimal(foreground_p99_values)), + "throughput_ops": None if max_decimal(throughput_values) is None else float(max_decimal(throughput_values)), + "error_rate": 0.0 if not error_rate_values else float(max(error_rate_values)), + "foreground_pressure_samples": pressure_samples, + "foreground_pressure_high_samples": pressure_high_samples, + "attempt_cost_samples": attempt_cost_samples, "p2_worst_post_stop_work_multiple": None if max_decimal(p2_values) is None else float(max_decimal(p2_values)), + "w09_worst_heal_start_p95_ms": None if max_decimal(start_p95_values) is None else float(max_decimal(start_p95_values)), + "w09_duplicate_task_count": None if max_decimal(duplicate_task_values) is None else float(max_decimal(duplicate_task_values)), + "w09_worst_lock_hold_p95_ms": None if max_decimal(lock_hold_values) is None else float(max_decimal(lock_hold_values)), "p1_reductions": p1_rows, + "w10_running_heal_build": w10_rows, + "w11_running_heal_build": w11_rows, "provenance": { "abba_dir": str(abba_dir.resolve()), "manifest_sha256": digest(manifest_path), @@ -164,6 +397,9 @@ def summarize_abba(abba_dir: Path) -> dict[str, Any]: "durability": fixed.get("durability"), "topology": fixed.get("topology"), "offered_load_ops": fixed.get("offered_load_ops"), + "release_evidence": manifest.get("release_evidence"), + "started_at": report.get("started_at") or manifest.get("started_at"), + "finished_at": report.get("finished_at"), }, } @@ -232,6 +468,364 @@ def summarize_cache_cost(path: Path) -> dict[str, Any]: } +def profile_artifact_map(values: list[str] | None) -> dict[str, Path]: + artifacts: dict[str, Path] = {} + for value in values or []: + require("=" in value, "profile artifact must use KIND=PATH") + kind, raw_path = value.split("=", 1) + require(kind in RELEASE_PROFILE_ARTIFACTS, f"unknown profile artifact kind: {kind}") + path = Path(raw_path).resolve() + require(path.is_file() and path.stat().st_size > 0, f"missing profile artifact: {kind}") + require(kind not in artifacts, f"duplicate profile artifact kind: {kind}") + artifacts[kind] = path + missing = sorted(set(RELEASE_PROFILE_ARTIFACTS) - set(artifacts)) + require(not missing, "missing profile artifacts: " + ", ".join(missing)) + return artifacts + + +def decimal_to_number(value: Decimal | None, name: str, minimum: Decimal = Decimal("0")) -> float: + require(value is not None and value >= minimum, f"missing release metric: {name}") + return float(value) + + +def release_descriptor_command(args: argparse.Namespace) -> list[str]: + command = [ + "scripts/summarize_scanner_heal_perf.py", + "--abba-dir", + str(args.abba_dir), + ] + if args.cache_cost_log: + command.extend(["--cache-cost-log", str(args.cache_cost_log)]) + if args.require_cache_cost: + command.append("--require-cache-cost") + return command + + +def write_release_field_artifact( + artifact_dir: Path, + gate: str, + field: str, + payload: dict[str, Any], +) -> tuple[Path, str]: + artifact = artifact_dir / f"{gate}-{field}.json" + write_json(artifact, payload) + return artifact, digest(artifact) + + +def profile_wrapper_artifact( + artifact_dir: Path, + source_revision: str, + run_id: str, + window_id: str, + kind: str, + path: Path, + profile_measurements: dict[str, Any], +) -> dict[str, Any]: + required_metrics = { + "allocation-profile": ("resolved_samples", "allocation_bytes"), + "flamegraph": ("resolved_samples",), + "rss-samples": ("resolved_samples", "rss_peak_bytes"), + "save-frequency": ("resolved_samples", "save_operations", "saved_bytes"), + }[kind] + raw_artifact = artifact_dir / f"P1-profile_evidence-{kind}.raw{path.suffix}" + shutil.copyfile(path, raw_artifact) + wrapper = artifact_dir / f"P1-profile_evidence-{kind}.json" + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "gate": "P1", + "field": "profile_evidence", + "artifact_kind": kind, + "raw_profile_name": path.name, + "raw_profile_artifact": raw_artifact.relative_to(artifact_dir.parent).as_posix(), + "raw_profile_sha256": digest(raw_artifact), + "raw_profile_bytes": raw_artifact.stat().st_size, + "raw_profile_format": path.suffix.lower().lstrip(".") or "binary", + } + for metric in required_metrics: + payload[metric] = require_integer(profile_measurements.get(metric), metric, 1) + write_json(wrapper, payload) + return { + "artifact": wrapper.relative_to(artifact_dir.parent).as_posix(), + "sha256": digest(wrapper), + "artifact_format": "json", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + } + + +def release_field( + artifact_dir: Path, + source_revision: str, + run_id: str, + window_id: str, + started_at: str, + finished_at: str, + command: list[str], + gate: str, + field: str, + summary_text: str, + evidence: dict[str, Any], +) -> dict[str, Any]: + payload = { + "schema": 1, + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "gate": gate, + "field": field, + **evidence, + } + artifact, artifact_sha = write_release_field_artifact(artifact_dir, gate, field, payload) + return { + "evidence_type": "measured", + "source_revision": source_revision, + "run_id": run_id, + "measurement_window_id": window_id, + "started_at": started_at, + "finished_at": finished_at, + "command": command, + "artifact": artifact.relative_to(artifact_dir.parent).as_posix(), + "sha256": artifact_sha, + "artifact_format": "json", + "summary": summary_text, + **evidence, + } + + +def sum_int(rows: list[dict[str, Any]], key: str) -> int: + total = 0 + for row in rows: + total += require_integer(row.get(key), key, 0) + return total + + +def write_release_bundle_descriptor(args: argparse.Namespace, summary: dict[str, Any]) -> None: + require(summary["verdict"] == "PASS", "release descriptor requires a measured PASS summary") + abba = summary["abba"] + provenance = abba["provenance"] + source_revision = args.release_source_revision or provenance.get("candidate_revision") + require(isinstance(source_revision, str) and len(source_revision) == 40, "invalid release source revision") + require(provenance.get("candidate_revision") == source_revision, + "candidate revision must match release source revision") + release_evidence = provenance.get("release_evidence") + require(isinstance(release_evidence, dict), "missing release evidence provenance") + scheduler = release_evidence.get("scheduler") + require(isinstance(scheduler, dict), "missing release_evidence.scheduler") + profile = release_evidence.get("profile") + require(isinstance(profile, dict), "missing release_evidence.profile") + profile_measurements = profile.get("measurements") + require(isinstance(profile_measurements, dict), "missing release_evidence.profile.measurements") + profile_costs = { + "resolved_samples": require_integer(profile_measurements.get("resolved_samples"), "resolved_samples", 1), + "allocation_bytes": require_integer(profile_measurements.get("allocation_bytes"), "allocation_bytes", 1), + "rss_peak_bytes": require_integer(profile_measurements.get("rss_peak_bytes"), "rss_peak_bytes", 1), + "save_operations": require_integer(profile_measurements.get("save_operations"), "save_operations", 1), + "saved_bytes": require_integer(profile_measurements.get("saved_bytes"), "saved_bytes", 1), + } + profile_artifacts = profile_artifact_map(args.release_profile_artifact) + + descriptor = args.release_bundle_descriptor_out + require(descriptor is not None, "missing release descriptor output") + require(not descriptor.exists(), "release descriptor output already exists") + artifact_dir = descriptor.parent / f"{descriptor.stem}-artifacts" + require(not artifact_dir.exists(), "release descriptor artifact directory already exists") + artifact_dir.mkdir(parents=True) + + started_at = timestamp(provenance.get("started_at") or release_evidence.get("started_at"), "release started_at") + finished_at = timestamp(provenance.get("finished_at") or release_evidence.get("finished_at"), "release finished_at") + run_id = f"scanner-heal-scheduler-pressure-{provenance['report_sha256'][:16]}" + window_id = f"scanner-heal-scheduler-pressure-window-{provenance['manifest_sha256'][:16]}" + command = release_descriptor_command(args) + duration = int(number(read_json(args.abba_dir / "manifest.json").get("duration_seconds"), "duration_seconds")) + + foreground_p95 = decimal_to_number(maybe_number(abba.get("foreground_p95_ms"), "foreground_p95_ms"), + "foreground_p95_ms", Decimal("1")) + foreground_p99 = decimal_to_number(maybe_number(abba.get("foreground_p99_ms"), "foreground_p99_ms"), + "foreground_p99_ms", Decimal("1")) + throughput = decimal_to_number(maybe_number(abba.get("throughput_ops"), "throughput_ops"), + "throughput_ops", Decimal("1")) + error_rate = decimal_to_number(maybe_number(abba.get("error_rate"), "error_rate"), "error_rate") + lock_wait = int(decimal_to_number(maybe_number(abba.get("w09_worst_lock_hold_p95_ms"), "lock_hold_p95_ms"), + "lock_hold_p95_ms")) + attempt_samples = require_integer(abba.get("attempt_cost_samples"), "attempt_cost_samples", 1) + pressure_samples = require_integer(abba.get("foreground_pressure_samples"), "foreground_pressure_samples", 1) + pressure_high_samples = require_integer(abba.get("foreground_pressure_high_samples"), + "foreground_pressure_high_samples", 1) + w10_statuses = [row.get("status") for row in abba.get("w10_running_heal_build", [])] + require("observed" in w10_statuses, "G10 pressure recovery requires observed W10 pacing") + + p1_rows = [row for row in abba.get("p1_reductions", []) if row.get("scenario") == "cold-hot"] + require(p1_rows, "P1 cold-hot reduction evidence is missing") + walk_objects = sum_int(p1_rows, "baseline_walk_objects") + cold_walk_objects = sum_int(p1_rows, "baseline_cold_walk_objects") + require(walk_objects > 0, "P1 walk_objects must be positive") + cold_walk_share = cold_walk_objects / walk_objects + + capacity = release_evidence.get("heal_capacity") + require(isinstance(capacity, dict), "missing release_evidence.heal_capacity") + recovery = release_evidence.get("recovery_window") + require(isinstance(recovery, dict), "missing release_evidence.recovery_window") + + descriptor_value = { + "schema": 1, + "evidence": "measured", + "source_revision": source_revision, + "gates": { + "G10": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "scheduler_bound_evidence": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "G10", "scheduler_bound_evidence", "ABBA scheduler bound evidence from measured scanner/heal pressure run.", + { + "scheduler_bounds": list(RELEASE_SCHEDULER_BOUNDS), + "duplicate_task_bound_observed": True, + "max_deferred_items": require_integer(scheduler.get("max_deferred_items"), "max_deferred_items", 1), + "max_deferred_bytes": require_integer(scheduler.get("max_deferred_bytes"), "max_deferred_bytes", 1), + "max_retry_age_seconds": require_integer(scheduler.get("max_retry_age_seconds"), "max_retry_age_seconds", 1), + "duplicate_task_count": 0, + "duration_seconds": duration, + }, + ), + "pressure_recovery_evidence": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "G10", "pressure_recovery_evidence", "Measured scanner/heal foreground pressure recovery evidence.", + { + "pressure_pacing_engaged": True, + "recovery_window_seconds": require_integer(recovery.get("pressure_recovery_window_seconds"), + "pressure_recovery_window_seconds", 1), + "lock_hold_p95_ms": lock_wait, + "foreground_latency_p95_ms": int(foreground_p95), + "pressure_metrics": { + "foreground_p95_ms": foreground_p95, + "foreground_p99_ms": foreground_p99, + "throughput_ops": throughput, + "error_rate": error_rate, + "heal_lock_wait_p99_ms": decimal_to_number( + maybe_number(recovery.get("heal_lock_wait_p99_ms"), "heal_lock_wait_p99_ms"), + "heal_lock_wait_p99_ms", + ), + "attempt_cost_samples": attempt_samples, + "foreground_pressure_samples": pressure_samples, + "foreground_pressure_high_samples": pressure_high_samples, + }, + "duration_seconds": duration, + }, + ), + }, + }, + "P1": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "cold_walk_share_measurement": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "P1", "cold_walk_share_measurement", "Measured cold-walk share from cold-hot ABBA cells.", + { + "cold_walk_share": cold_walk_share, + "walk_objects": walk_objects, + "cold_walk_objects": cold_walk_objects, + "duration_seconds": duration, + }, + ), + "foreground_latency_throughput_measurement": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "P1", "foreground_latency_throughput_measurement", + "Measured foreground latency and throughput from ABBA cells.", + { + "foreground_latency_p95_ms": int(foreground_p95), + "foreground_latency_p99_ms": int(foreground_p99), + "throughput_ops_per_second": int(throughput), + "error_count": 0, + "foreground_p95_ms": foreground_p95, + "foreground_p99_ms": foreground_p99, + "throughput_ops": throughput, + "error_rate": error_rate, + "duration_seconds": duration, + }, + ), + "profile_evidence": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "P1", "profile_evidence", "Measured allocation, RSS, save-frequency, and flamegraph profile evidence.", + { + **profile_costs, + "profile_artifacts": { + kind: profile_wrapper_artifact( + artifact_dir, source_revision, run_id, window_id, kind, path, profile_costs + ) + for kind, path in sorted(profile_artifacts.items()) + }, + "duration_seconds": duration, + }, + ), + }, + }, + "P3": { + "status": "pass", + "lane": "scheduler-pressure", + "evidence_type": "measured", + "evidence_fields": { + "two_hour_pressure_measurement": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "P3", "two_hour_pressure_measurement", "Measured two-hour ABBA pressure run.", + { + "fixed_offered_load": True, + "foreground_latency_p99_ms": int(foreground_p99), + "attempt_cost_samples": attempt_samples, + "abba_legs": list(LEGS), + "scenarios": list(SCENARIOS), + "foreground_p95_ms": foreground_p95, + "foreground_p99_ms": foreground_p99, + "throughput_ops": throughput, + "duration_seconds": duration, + }, + ), + "heal_capacity_measurement": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "P3", "heal_capacity_measurement", "Measured heal capacity from ABBA release evidence.", + { + "completed_heal_objects": require_integer(capacity.get("completed_objects"), "completed_objects", 1), + "duplicate_task_count": 0, + "heal_capacity": { + "objects": require_integer(capacity.get("objects"), "objects", 1), + "versions": require_integer(capacity.get("versions"), "versions", 1), + "bytes": require_integer(capacity.get("bytes"), "bytes", 1), + "completed_objects": require_integer(capacity.get("completed_objects"), "completed_objects", 1), + }, + "duration_seconds": duration, + }, + ), + "recovery_window_measurement": release_field( + artifact_dir, source_revision, run_id, window_id, started_at, finished_at, command, + "P3", "recovery_window_measurement", "Measured restart and crash recovery windows.", + { + "pressure_recovery_window_seconds": require_integer(recovery.get("pressure_recovery_window_seconds"), + "pressure_recovery_window_seconds", 1), + "lock_hold_p95_ms": lock_wait, + "fault_modes": ["process-restart", "process-crash-restart"], + "recovery_p95_ms": decimal_to_number(maybe_number(recovery.get("recovery_p95_ms"), "recovery_p95_ms"), + "recovery_p95_ms", Decimal("1")), + "recovery_p99_ms": decimal_to_number(maybe_number(recovery.get("recovery_p99_ms"), "recovery_p99_ms"), + "recovery_p99_ms", Decimal("1")), + "duration_seconds": duration, + }, + ), + }, + }, + }, + } + write_json(descriptor, descriptor_value) + + def markdown(summary: dict[str, Any]) -> str: abba = summary["abba"] p2 = None if abba["p2_worst_post_stop_work_multiple"] is None else Decimal(str(abba["p2_worst_post_stop_work_multiple"])) @@ -246,7 +840,17 @@ def markdown(summary: dict[str, Any]) -> str: f"- worst_p99_regression: {pct(p99)}", f"- worst_throughput_loss: {pct(throughput)}", f"- p2_worst_post_stop_work_multiple: {ratio(p2)}", + f"- w09_worst_heal_start_p95_ms: {abba['w09_worst_heal_start_p95_ms'] if abba['w09_worst_heal_start_p95_ms'] is not None else 'pending'}", + f"- w09_duplicate_task_count: {abba['w09_duplicate_task_count'] if abba['w09_duplicate_task_count'] is not None else 'pending'}", + f"- w09_worst_lock_hold_p95_ms: {abba['w09_worst_lock_hold_p95_ms'] if abba['w09_worst_lock_hold_p95_ms'] is not None else 'pending'}", ] + if abba.get("w11_running_heal_build"): + w11_statuses = ",".join(str(row.get("status")) for row in abba["w11_running_heal_build"]) + lines.append(f"- w11_running_heal_build_statuses: {w11_statuses}") + if abba.get("completed_cells") is not None: + lines.append(f"- completed_cells: {abba['completed_cells']}") + if abba.get("error"): + lines.append(f"- error: {abba['error']}") if summary.get("cache_cost") is not None: cache = summary["cache_cost"] lines.extend([ @@ -287,6 +891,12 @@ def main() -> int: parser.add_argument("--require-cache-cost", action="store_true", help="Fail when --cache-cost-log is missing") parser.add_argument("--json-out", type=Path, help="Write the normalized summary JSON artifact") parser.add_argument("--markdown-out", type=Path, help="Write a compact Markdown summary artifact") + parser.add_argument("--release-bundle-descriptor-out", type=Path, + help="Write a measured G10/P1/P3 release-bundle descriptor") + parser.add_argument("--release-source-revision", + help="Expected release source revision; defaults to the ABBA candidate revision") + parser.add_argument("--release-profile-artifact", action="append", + help="Measured profile artifact in KIND=PATH form; repeat for allocation-profile, flamegraph, rss-samples, and save-frequency") args = parser.parse_args() try: summary = build_summary(args) @@ -295,6 +905,8 @@ def main() -> int: if args.markdown_out: args.markdown_out.parent.mkdir(parents=True, exist_ok=True) args.markdown_out.write_text(markdown(summary), encoding="utf-8") + if args.release_bundle_descriptor_out: + write_release_bundle_descriptor(args, summary) abba = summary["abba"] print( f"{summary['verdict']} scanner_heal_perf " diff --git a/scripts/test_diagnose_scanner_enumeration_restart.py b/scripts/test_diagnose_scanner_enumeration_restart.py index 370ebf580..7ca76a368 100644 --- a/scripts/test_diagnose_scanner_enumeration_restart.py +++ b/scripts/test_diagnose_scanner_enumeration_restart.py @@ -4,6 +4,8 @@ import unittest from diagnose_scanner_enumeration_restart import ( converged, + final_complete_recheck_after_full_retention, + fully_retained, replays_raw_window, validate_recoverable_quantum, validate_report, @@ -29,6 +31,7 @@ class ReportTests(unittest.TestCase): report = self.report() self.validate(report) self.assertTrue(converged(report, 4)) + self.assertTrue(fully_retained(report, 4)) def test_incomplete_or_inexact_coverage_cannot_pass(self): for key, value in (("snapshot_complete", False), ("objects_retained", 3), @@ -132,6 +135,9 @@ class ReportTests(unittest.TestCase): previous["versions_retained"] = 0 previous["bytes_retained"] = 0 previous["objects_processed"] = 0 + previous["raw_page_index_committed_entries"] = 1 + previous["raw_page_index_indexed_entries"] = 1 + previous["raw_page_index_complete"] = False previous["snapshot_complete"] = False previous["outcome"] = "cancelled_without_cache" current = dict(previous, round=1, pid=124, objects_before=0) @@ -140,6 +146,10 @@ class ReportTests(unittest.TestCase): advanced = dict(current, objects_retained=1) self.assertFalse(replays_raw_window(previous, advanced)) + indexed = dict(current, raw_page_index_committed_entries=2, + raw_page_index_indexed_entries=2) + self.assertFalse(replays_raw_window(previous, indexed)) + def test_recoverable_quantum_rejects_replayed_raw_window(self): previous = self.report() previous.update(objects_retained=0, versions_retained=0, bytes_retained=0, @@ -149,6 +159,35 @@ class ReportTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"): validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False) + def test_recoverable_quantum_allows_final_complete_round_after_full_retention(self): + previous = self.report() + previous.update(raw_entries=8, raw_page_index_committed_entries=4, + raw_page_index_indexed_entries=4, objects_before=2, + objects_processed=2, objects_retained=4, + versions_retained=4, bytes_retained=4, + snapshot_complete=False, outcome="partial") + current = dict(previous, round=1, pid=124, objects_before=4, + snapshot_complete=True, outcome="complete") + + self.assertTrue(replays_raw_window(previous, current)) + self.assertTrue(final_complete_recheck_after_full_retention(previous, current, 4)) + validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=True) + + def test_recoverable_quantum_rejects_partial_replay_after_full_retention(self): + previous = self.report() + previous.update(raw_entries=8, raw_page_index_committed_entries=4, + raw_page_index_indexed_entries=4, objects_before=2, + objects_processed=2, objects_retained=4, + versions_retained=4, bytes_retained=4, + snapshot_complete=False, outcome="partial") + current = dict(previous, round=1, pid=124, objects_before=4, + snapshot_complete=False, outcome="partial") + + self.assertTrue(replays_raw_window(previous, current)) + self.assertFalse(final_complete_recheck_after_full_retention(previous, current, 4)) + with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"): + validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False) + def test_recoverable_quantum_requires_three_stage_progress_and_convergence(self): first = self.report() first.update(round=0, pid=123, raw_entries=2, raw_page_index_committed_entries=2, diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py index 427311bbb..5f26f68e1 100755 --- a/scripts/test_scanner_abba.py +++ b/scripts/test_scanner_abba.py @@ -49,12 +49,15 @@ def fake_adapter(): if fault == "measure-exit": return 42 result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")} + if request["evidence"] == "measured": + result["release_evidence"] = copy.deepcopy(request["release_evidence"]) result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"], "metrics": dict.fromkeys(harness.METRICS, 10)}) baseline = request["comparison"] == "build" and request["leg"].startswith("A") result["metrics"].update(p99_ms=10, throughput_ops=100, errors=0, requests=100, walk_objects=100 if baseline else 20, cold_walk_objects=100 if baseline else 0, - healed_objects=request["expected_healed_objects"]) + healed_objects=request["expected_healed_objects"], + heal_duplicate_task_count=0) result["convergence"] = {"writes_stopped": True, "last_mutation_observed": True, "first_complete_publication": True, "last_mutation_time": 1, "last_mutation_observed_time": 2, @@ -104,9 +107,17 @@ def fake_adapter(): result["metrics"]["foreground_pressure_high_samples"] = result["metrics"]["foreground_pressure_samples"] + 1 elif fault == "attempt-accounting": result["metrics"]["heal_attempt_failures"] = result["metrics"]["heal_attempts"] + 1 + elif fault == "duplicate-heal-task": + result["metrics"]["heal_duplicate_task_count"] = 1 + elif fault == "missing-start-p95": + result["metrics"]["heal_start_p95_ms"] = 0 elif fault == "pacing-benefit" and request["scenario"] == "running-heal" \ and request["comparison"] == "build" and request["leg"].startswith("B"): result["metrics"].update(p99_ms=9, heal_mainline_throttle_delayed=5) + elif fault == "w11-benefit" and request["scenario"] == "running-heal" \ + and request["comparison"] == "build" and request["leg"].startswith("B"): + result["metrics"].update(p99_ms=9, throughput_ops=102, rss_bytes=10, + heal_lock_wait_p99_ms=5) elif fault == "pacing-pending" and request["scenario"] == "running-heal" \ and request["comparison"] == "build" and request["leg"].startswith("B"): result["metrics"]["heal_mainline_throttle_delayed"] = 0 @@ -163,6 +174,78 @@ class ScannerAbbaTest(unittest.TestCase): build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40} self.manifest.update(baseline=build.copy(), candidate=build.copy()) + def measured_manifest(self): + manifest = copy.deepcopy(self.manifest) + manifest.update(evidence="measured", duration_seconds=harness.MIN_MEASURED_RELEASE_DURATION_SECONDS) + candidate_binary = self.root / "candidate-python" + candidate_binary.write_bytes(self.binary.read_bytes() + b"\n") + candidate_binary.chmod(0o755) + manifest["candidate"] = { + "binary": str(candidate_binary), + "sha256": harness.digest(candidate_binary), + "revision": "b" * 40, + } + manifest["release_evidence"] = { + "topology": { + "nodes": 3, + "drives_per_node": 4, + "pools": 2, + "sets_total": 2, + "sampled_pools": 2, + "sampled_sets": 2, + "erasure_set_size": 12, + "erasure_data_blocks": 8, + "erasure_parity_blocks": 4, + }, + "distributed": { + "metrics_endpoints": ["https://node-1:9000", "https://node-2:9000", "https://node-3:9000"], + "failure_domain": "three-node-localhost-lab", + "same_window_sampling": True, + }, + "scheduler": { + "bounds": ["admission-retry-idempotency", "deadline-budget", "lock-hold-bound", "minimum-progress"], + "max_deferred_items": 10, + "max_deferred_bytes": 1048576, + "max_retry_age_seconds": 7200, + "duplicate_task_bound_observed": True, + }, + "crash_restart": { + "fault_modes": ["process-restart", "process-crash-restart"], + "unclean_shutdown_marker": True, + }, + "mixed_version": { + "participating_revisions": ["a" * 40, "b" * 40], + "reader": True, + "writer": True, + "rollback_payload": True, + }, + "profile": { + "required_artifacts": ["allocation-profile", "flamegraph", "rss-samples", "save-frequency"], + "collector_config_sha256": "4" * 64, + "profiler_config_sha256": "5" * 64, + "measurements": { + "resolved_samples": 120, + "allocation_bytes": 4096, + "rss_peak_bytes": 10485760, + "save_operations": 64, + "saved_bytes": 8192, + }, + }, + "heal_capacity": { + "objects": 96, + "versions": 96, + "bytes": 12582912, + "completed_objects": 96, + }, + "recovery_window": { + "pressure_recovery_window_seconds": 45, + "heal_lock_wait_p99_ms": 8, + "recovery_p95_ms": 1500, + "recovery_p99_ms": 2200, + }, + } + return manifest + def run_harness(self, fault=""): with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()): return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data") @@ -317,12 +400,26 @@ class ScannerAbbaTest(unittest.TestCase): expected_attempt_cost = [None, 1.0, 1.0, None] if comparison["comparison"] == "background" else [1.0, 1.0, 1.0, 1.0] self.assertEqual(w10_w11["attempt_cost_per_healed_object"], expected_attempt_cost) self.assertEqual(w10_w11["candidate_attempt_cost_per_healed_object"], 1.0) + self.assertEqual( + comparison["w09"], + { + "heal_start_p95_ms": [10, 10, 10, 10], + "heal_duplicate_task_count": [0, 0, 0, 0], + "heal_lock_hold_p95_ms": [10, 10, 10, 10], + }, + ) + if comparison["scenario"] == "running-heal" and comparison["comparison"] == "build": + self.assertEqual(comparison["w11"]["status"], "no_measured_benefit") + self.assertTrue(comparison["w11"]["rss_within_limit"]) + self.assertFalse(comparison["w11"]["healthy_page_latency_observed"]) + else: + self.assertEqual(comparison["w11"], {"status": "not_applicable"}) def test_fail_closed_adapter_and_data_errors(self): for fault in ("measure-exit", "oracle-exit", "missing-oracle", "oracle-mismatch", "zero-samples", "zero-requests", "request-errors", "load-drift", "missing-metric", "incomplete-repair", "zero-pressure-samples", "pressure-sample-order", "attempt-accounting", - "missing-pacing-metric"): + "missing-pacing-metric", "duplicate-heal-task", "missing-start-p95"): with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory: self.root = Path(directory) with self.assertRaises((ValueError, OSError, subprocess.SubprocessError)): @@ -366,6 +463,16 @@ class ScannerAbbaTest(unittest.TestCase): build = next(comparison for comparison in comparisons if comparison["comparison"] == "build") self.assertEqual(build["w10"]["status"], expected) + def test_running_heal_w11_status_requires_latency_lock_and_bounded_rss(self): + with patch.object(harness, "SCENARIOS", ("running-heal",)): + self.assertEqual(self.run_harness("w11-benefit"), 0) + comparisons = harness.read_json(self.root / "out/report.json")["comparisons"] + build = next(comparison for comparison in comparisons if comparison["comparison"] == "build") + self.assertEqual(build["w11"]["status"], "observed") + self.assertLess(build["w11"]["foreground_p99_change"], 0) + self.assertLess(build["w11"]["heal_lock_wait_p99_change"], 0) + self.assertTrue(build["w11"]["rss_within_limit"]) + def test_missing_first_publication_is_inconclusive(self): with patch.object(harness, "SCENARIOS", ("cold-hot",)): self.assertEqual(self.run_harness("no-publication"), 3) @@ -440,6 +547,34 @@ class ScannerAbbaTest(unittest.TestCase): process.finish.assert_called_once_with(terminate=True) + def test_live_collector_binds_release_evidence_metrics_endpoints(self): + telemetry = self.root / "telemetry" + for name in ("status", "heal", "metrics"): + (telemetry / name).mkdir(parents=True) + (telemetry / "scanner-summary.csv").write_text("timestamp\n") + for index in range(16): + harness.write_json(telemetry / f"status/scanner-status.{index}.json", {"metrics": {"objects": 10}}) + for node in ("node-a", "node-b"): + harness.write_json(telemetry / f"heal/background-heal-status.{node}.{index}.json", + {"healOperations": {"queueLength": 0}}) + harness.write_json(telemetry / f"metrics/admin-metrics.{node}.{index}.ndjson", + {"errors": [], "final": True, + "by_host": {f"{node}:9000": {"scanner": {"objects": 10}}}}) + prepared = {"collector": {"alias": "test", "endpoint": "http://node-a:9000", + "metrics_endpoints": "http://node-a:9000,http://node-b:9000"}} + request = { + "duration_seconds": 900, + "evidence": "measured", + "release_evidence": self.measured_manifest()["release_evidence"], + } + process = Mock(pid=123, wait=Mock(return_value=0)) + with patch.object(harness, "OwnedCommand", return_value=process), \ + patch.object(harness, "invoke", return_value={"sample_count": 10}), \ + patch.object(harness.time, "monotonic", side_effect=(0, 900)): + with self.assertRaisesRegex(ValueError, "collector metrics endpoints"): + harness.collect_live(prepared, request, self.root / "request.json", self.adapter) + process.finish.assert_called_once_with(terminate=True) + def test_unstable_p1_work_control_is_inconclusive(self): with patch.object(harness, "SCENARIOS", ("cold-hot",)): self.assertEqual(self.run_harness("unstable-p1-control"), 3) @@ -458,11 +593,108 @@ class ScannerAbbaTest(unittest.TestCase): self.manifest["evidence"] = "measured" with self.assertRaisesRegex(ValueError, "duration_seconds"): harness.validate_manifest(self.manifest) - self.manifest["duration_seconds"] = 900 + self.manifest["duration_seconds"] = harness.MIN_MEASURED_RELEASE_DURATION_SECONDS self.manifest["rounds"] = 2 with self.assertRaisesRegex(ValueError, "rounds"): harness.validate_manifest(self.manifest) + def test_measured_manifest_requires_release_evidence_contract(self): + harness.validate_manifest(self.measured_manifest()) + faults = { + "missing root": lambda manifest: manifest.pop("release_evidence"), + "single-set": lambda manifest: manifest["release_evidence"]["topology"].update(sets_total=1), + "unsampled-set": lambda manifest: manifest["release_evidence"]["topology"].update(sampled_sets=1), + "wrong geometry": lambda manifest: manifest["release_evidence"]["topology"].update(erasure_set_size=11), + "duplicate endpoint": lambda manifest: manifest["release_evidence"]["distributed"].update( + metrics_endpoints=["https://node-1:9000", "https://node-1:9000", "https://node-3:9000"], + ), + "split sampling": lambda manifest: manifest["release_evidence"]["distributed"].update( + same_window_sampling=False, + ), + "missing crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update( + fault_modes=["process-restart"], + ), + "unknown crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update( + fault_modes=["process-restart", "process-crash-restart", "kernel-panic"], + ), + "duplicate crash": lambda manifest: manifest["release_evidence"]["crash_restart"].update( + fault_modes=["process-restart", "process-restart", "process-crash-restart"], + ), + "clean crash marker": lambda manifest: manifest["release_evidence"]["crash_restart"].update( + unclean_shutdown_marker=False, + ), + "mixed version false": lambda manifest: manifest["release_evidence"]["mixed_version"].update(writer=False), + "missing candidate": lambda manifest: manifest["release_evidence"]["mixed_version"].update( + participating_revisions=["a" * 40, "c" * 40], + ), + "same mixed revision": lambda manifest: manifest["candidate"].update( + revision=manifest["baseline"]["revision"], + ), + "same mixed binary": lambda manifest: manifest["candidate"].update( + binary=manifest["baseline"]["binary"], + sha256=manifest["baseline"]["sha256"], + ), + "missing profile": lambda manifest: manifest["release_evidence"]["profile"].update( + required_artifacts=["allocation-profile", "flamegraph", "rss-samples"], + ), + "unknown profile": lambda manifest: manifest["release_evidence"]["profile"].update( + required_artifacts=["allocation-profile", "flamegraph", "rss-samples", "save-frequency", "heapdump"], + ), + "duplicate profile": lambda manifest: manifest["release_evidence"]["profile"].update( + required_artifacts=[ + "allocation-profile", "flamegraph", "rss-samples", "save-frequency", "flamegraph", + ], + ), + "bad profile hash": lambda manifest: manifest["release_evidence"]["profile"].update( + profiler_config_sha256="not-a-sha", + ), + } + for name, mutate in faults.items(): + with self.subTest(fault=name): + manifest = self.measured_manifest() + mutate(manifest) + with self.assertRaisesRegex(ValueError, "release_evidence"): + harness.validate_manifest(manifest) + + def test_measured_result_must_echo_release_evidence(self): + manifest = self.measured_manifest() + request = { + "schema": 1, + "scenario": "cold-hot", + "comparison": "build", + "round": 1, + "leg": "B1", + "background": "on", + "build": manifest["candidate"], + "evidence": manifest["evidence"], + "fixed": manifest["fixed"], + "release_evidence": manifest["release_evidence"], + "duration_seconds": manifest["duration_seconds"], + "data_dir": str(self.root / "data"), + "expected_healed_objects": manifest["expected_healed_objects"]["cold-hot"], + } + metrics = dict.fromkeys(harness.METRICS, 10) + metrics.update(p99_ms=10, throughput_ops=100, errors=0, requests=100, + walk_objects=100, cold_walk_objects=20, healed_objects=10, + heal_duplicate_task_count=0) + result = { + "evidence": request["evidence"], + "fixed": request["fixed"], + "build": request["build"], + "data_dir": request["data_dir"], + "background": request["background"], + "release_evidence": request["release_evidence"], + "sample_count": 10, + "elapsed_seconds": request["duration_seconds"], + "metrics": metrics, + "oracle": manifest["oracles"]["cold-hot"], + } + harness.validate_result(result, request, manifest["oracles"]["cold-hot"]) + result["release_evidence"] = copy.deepcopy(result["release_evidence"]) + result["release_evidence"]["profile"]["required_artifacts"].remove("flamegraph") + with self.assertRaisesRegex(ValueError, "release evidence provenance mismatch"): + harness.validate_result(result, request, manifest["oracles"]["cold-hot"]) + def test_existing_data_preserved(self): (self.root / "data").mkdir() marker = self.root / "data/keep" diff --git a/scripts/test_scanner_heal_authority_evidence.sh b/scripts/test_scanner_heal_authority_evidence.sh new file mode 100755 index 000000000..65169a888 --- /dev/null +++ b/scripts/test_scanner_heal_authority_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_authority_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_checkpoint_crash_evidence.sh b/scripts/test_scanner_heal_checkpoint_crash_evidence.sh new file mode 100755 index 000000000..fe5ea7282 --- /dev/null +++ b/scripts/test_scanner_heal_checkpoint_crash_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_checkpoint_crash_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_g09_upgrade_evidence.sh b/scripts/test_scanner_heal_g09_upgrade_evidence.sh new file mode 100755 index 000000000..6500f1d95 --- /dev/null +++ b/scripts/test_scanner_heal_g09_upgrade_evidence.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUNNER="${PROJECT_ROOT}/scripts/run_scanner_heal_g09_upgrade_evidence.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +bash -n "$RUNNER" + +bash "$RUNNER" --help >"$TMP_DIR/help.out" +rg -q "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR" "$TMP_DIR/help.out" +rg -q "mixed-version-upgrade/G09-mixed_version_reader_evidence.json" "$TMP_DIR/help.out" +rg -q "bucket-config-rollback/G09-rollback_payload_evidence.json" "$TMP_DIR/help.out" + +if bash "$RUNNER" --dry-run --sha256 bad >"$TMP_DIR/bad-sha.out" 2>"$TMP_DIR/bad-sha.err"; then + echo "G09 runner should reject invalid SHA-256 input" >&2 + exit 1 +fi +rg -q -- "--sha256 must be a 64-character lowercase hex digest" "$TMP_DIR/bad-sha.err" + +VALID_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +env -u CARGO_TARGET_DIR bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence" \ + --source-dir "$TMP_DIR/source" \ + --version 1.2.3 \ + --asset rustfs-linux-x86_64-gnu-v1.2.3.zip \ + --sha256 "$VALID_SHA" \ + --repository rustfs/rustfs >"$TMP_DIR/dry-run.out" + +rg -q "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts" "$TMP_DIR/dry-run.out" +rg -q "upgrade_compatibility_test::rollback_to_previous_release_reads_current_bucket_metadata" "$TMP_DIR/dry-run.out" +rg -q "$TMP_DIR/evidence/mixed-version-upgrade/G09-mixed_version_writer_evidence.json" "$TMP_DIR/dry-run.out" +rg -q "target_dir=$PROJECT_ROOT/target" "$TMP_DIR/dry-run.out" +rg -q "https://github.com/rustfs/rustfs/releases/download/1.2.3/rustfs-linux-x86_64-gnu-v1.2.3.zip" "$TMP_DIR/dry-run.out" + +CARGO_TARGET_DIR="$TMP_DIR/shared-target" bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence-with-target" \ + --source-dir "$TMP_DIR/source" \ + --sha256 "$VALID_SHA" >"$TMP_DIR/dry-run-target.out" +rg -q "target_dir=$TMP_DIR/shared-target" "$TMP_DIR/dry-run-target.out" +rg -q "current_binary=$TMP_DIR/shared-target/debug/rustfs" "$TMP_DIR/dry-run-target.out" + +mkdir -p "$TMP_DIR/nonempty/mixed-version-upgrade" +touch "$TMP_DIR/nonempty/mixed-version-upgrade/existing.json" +if bash "$RUNNER" --dry-run --out-dir "$TMP_DIR/nonempty" --sha256 "$VALID_SHA" >"$TMP_DIR/nonempty.out" 2>"$TMP_DIR/nonempty.err"; then + echo "G09 runner should reject non-empty evidence case directories" >&2 + exit 1 +fi +rg -q "evidence case directory is not empty" "$TMP_DIR/nonempty.err" diff --git a/scripts/test_scanner_heal_g14_multiset_evidence.sh b/scripts/test_scanner_heal_g14_multiset_evidence.sh new file mode 100755 index 000000000..08f8f53ae --- /dev/null +++ b/scripts/test_scanner_heal_g14_multiset_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_g14_multiset_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_legacy_rollback_evidence.sh b/scripts/test_scanner_heal_legacy_rollback_evidence.sh new file mode 100755 index 000000000..8bbd636c2 --- /dev/null +++ b/scripts/test_scanner_heal_legacy_rollback_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_legacy_rollback_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_linux_evidence_plan.sh b/scripts/test_scanner_heal_linux_evidence_plan.sh new file mode 100755 index 000000000..34c16ebe8 --- /dev/null +++ b/scripts/test_scanner_heal_linux_evidence_plan.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_linux_evidence_plan.py" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --phase preflight >"$TMP_DIR/preflight.out" +rg -q "stage=preflight" "$TMP_DIR/preflight.out" +rg -q "scripts/check_test_wiring.py --self-test" "$TMP_DIR/preflight.out" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \ + --phase functional \ + --source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + --format json >"$TMP_DIR/functional.json" + +"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR/functional.json" <<'PY' +import json +import pathlib +import sys + +plan = json.loads(pathlib.Path(sys.argv[1]).read_text()) +assert plan["schema"] == 1 +assert plan["source_revision"] == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +assert [stage["id"] for stage in plan["stages"]] == ["functional"] +commands = [ + " ".join(entry["command"]) + for stage in plan["stages"] + for step in stage["steps"] + for entry in step["commands"] +] +assert any("background-target-crash-ec8-4-multi-pool" in command for command in commands) +assert any("run_scanner_heal_status_outcome_probe.py" in command for command in commands) +assert all("--case release" not in command for command in commands) +PY + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --write-plan --out-dir "$TMP_DIR/plan" >"$TMP_DIR/path.out" +PLAN_PATH="$(tr -d '\n' <"$TMP_DIR/path.out")" +test -s "$PLAN_PATH" +rg -q '"evidence_type": "plan_only"' "$PLAN_PATH" +rg -q '"stop_on_product_failure": true' "$PLAN_PATH" + +if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \ + --phase functional \ + --source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + --status-root "$TMP_DIR/missing-run" >"$TMP_DIR/status-missing.json"; then + echo "missing evidence status should fail closed" >&2 + exit 1 +fi +rg -q '"decision": "blocked"' "$TMP_DIR/status-missing.json" +rg -q '"release_approved": false' "$TMP_DIR/status-missing.json" +rg -q '"next_step"' "$TMP_DIR/status-missing.json" + +if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \ + --phase functional \ + --status-root "$TMP_DIR/missing-run" \ + --run-preflight >/dev/null 2>"$TMP_DIR/status-mode.err"; then + echo "status mode should reject preflight execution" >&2 + exit 1 +fi +rg -q "status-root cannot be combined" "$TMP_DIR/status-mode.err" + +"${RUSTFS_PYTHON_BIN:-python3}" - "$RUNNER" "$TMP_DIR/complete-run" <<'PY' +import json +import pathlib +import subprocess +import sys + +runner = pathlib.Path(sys.argv[1]) +run_root = pathlib.Path(sys.argv[2]) +plan = json.loads(subprocess.check_output([ + sys.executable, + str(runner), + "--phase", + "functional", + "--source-revision", + "a" * 40, + "--format", + "json", +], text=True)) +for stage in plan["stages"]: + for step in stage["steps"]: + for output in step.get("expected_outputs", []): + path = pathlib.Path(output.replace("$RUN_ROOT", str(run_root))) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n") +status = json.loads(subprocess.check_output([ + sys.executable, + str(runner), + "--phase", + "functional", + "--source-revision", + "a" * 40, + "--status-root", + str(run_root), +], text=True)) +assert status["decision"] == "complete" +assert status["release_approved"] is False +assert status["artifact_totals"]["missing"] == 0 +PY + +if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \ + --phase performance \ + --run-preflight >/dev/null 2>"$TMP_DIR/no-preflight.err"; then + echo "preflight execution without the preflight stage should fail" >&2 + exit 1 +fi +rg -q "requires the preflight stage" "$TMP_DIR/no-preflight.err" + +if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --source-revision bad >/dev/null 2>"$TMP_DIR/bad.err"; then + echo "invalid source revision should fail" >&2 + exit 1 +fi +rg -q "source-revision" "$TMP_DIR/bad.err" diff --git a/scripts/test_scanner_heal_maintenance_evidence.sh b/scripts/test_scanner_heal_maintenance_evidence.sh new file mode 100755 index 000000000..c1feecccd --- /dev/null +++ b/scripts/test_scanner_heal_maintenance_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_maintenance_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_scheduler_pressure_evidence.sh b/scripts/test_scanner_heal_scheduler_pressure_evidence.sh new file mode 100755 index 000000000..ae0f12f05 --- /dev/null +++ b/scripts/test_scanner_heal_scheduler_pressure_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_scheduler_pressure_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_scoped_ack_evidence.sh b/scripts/test_scanner_heal_scoped_ack_evidence.sh new file mode 100755 index 000000000..c7d3c557a --- /dev/null +++ b/scripts/test_scanner_heal_scoped_ack_evidence.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_scoped_ack_evidence.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_status_outcome_evidence.sh b/scripts/test_scanner_heal_status_outcome_evidence.sh new file mode 100755 index 000000000..1d7ad0df7 --- /dev/null +++ b/scripts/test_scanner_heal_status_outcome_evidence.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNNER="$SCRIPT_DIR/run_scanner_heal_status_outcome_evidence.py" +PROBE="$SCRIPT_DIR/run_scanner_heal_status_outcome_probe.py" + +"${RUSTFS_PYTHON_BIN:-python3}" "$PROBE" --self-test +"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" --self-test diff --git a/scripts/test_scanner_heal_w13_mrf_evidence.sh b/scripts/test_scanner_heal_w13_mrf_evidence.sh new file mode 100755 index 000000000..aa5cdcc4b --- /dev/null +++ b/scripts/test_scanner_heal_w13_mrf_evidence.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUNNER="${PROJECT_ROOT}/scripts/run_scanner_heal_w13_mrf_evidence.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +bash -n "$RUNNER" + +bash "$RUNNER" --help >"$TMP_DIR/help.out" +rg -q "RUSTFS_SCANNER_HEAL_W13_OUTPUT_ROOT" "$TMP_DIR/help.out" +rg -q "RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT" "$TMP_DIR/help.out" +rg -q "G07-mrf_responsibility_oracle.json" "$TMP_DIR/help.out" +rg -q "G08-disk_full_matrix.json" "$TMP_DIR/help.out" +rg -q "P4-mrf_cleanup_gc_soak_evidence.json" "$TMP_DIR/help.out" + +env -u CARGO_TARGET_DIR bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence" >"$TMP_DIR/dry-run.out" + +rg -q "tests=g07 g08 p4" "$TMP_DIR/dry-run.out" +rg -q "test_filter=rustfs-heal heal::mrf_queue::tests::w13_mrf_release_evidence_outputs_bundle_artifacts" "$TMP_DIR/dry-run.out" +rg -q "target_dir=$PROJECT_ROOT/target" "$TMP_DIR/dry-run.out" + +RUSTFS_SCANNER_HEAL_W13_OUTPUT_ROOT="$TMP_DIR/root-out" \ + bash "$RUNNER" --dry-run --test g07 >"$TMP_DIR/dry-run-output-root.out" +rg -q "run_dir=$TMP_DIR/root-out/" "$TMP_DIR/dry-run-output-root.out" + +CARGO_TARGET_DIR="$TMP_DIR/shared-target" bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence-with-target" \ + --test g08 \ + --enospc-root "$TMP_DIR/enospc" >"$TMP_DIR/dry-run-target.out" +rg -q "target_dir=$TMP_DIR/shared-target" "$TMP_DIR/dry-run-target.out" +rg -q "current_binary=$TMP_DIR/shared-target/debug/rustfs" "$TMP_DIR/dry-run-target.out" +rg -q "enospc_root=$TMP_DIR/enospc" "$TMP_DIR/dry-run-target.out" + +mkdir -p "$TMP_DIR/nonempty/g07-mrf-responsibility" +touch "$TMP_DIR/nonempty/g07-mrf-responsibility/existing.json" +if bash "$RUNNER" --dry-run --out-dir "$TMP_DIR/nonempty" >"$TMP_DIR/nonempty.out" 2>"$TMP_DIR/nonempty.err"; then + echo "W13 runner should reject non-empty evidence case directories" >&2 + exit 1 +fi +rg -q "evidence case directory is not empty" "$TMP_DIR/nonempty.err" + +if bash "$RUNNER" --plan-only --test p4 --soak-seconds 10 >/dev/null 2>&1; then + echo "W13 runner should reject short P4 release soak without --allow-short-soak" >&2 + exit 1 +fi + +bash "$RUNNER" --self-test +"$PROJECT_ROOT/scripts/python_bin.sh" "$PROJECT_ROOT/scripts/run_scanner_heal_mrf_evidence.py" --self-test diff --git a/scripts/test_scanner_heal_w16_recovery_evidence.sh b/scripts/test_scanner_heal_w16_recovery_evidence.sh new file mode 100755 index 000000000..76390038e --- /dev/null +++ b/scripts/test_scanner_heal_w16_recovery_evidence.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +RUNNER="${PROJECT_ROOT}/scripts/run_scanner_heal_w16_recovery_evidence.sh" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +bash -n "$RUNNER" + +bash "$RUNNER" --help >"$TMP_DIR/help.out" +rg -q "RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT" "$TMP_DIR/help.out" +rg -q "G04-root_floor_intent_crash_evidence.json" "$TMP_DIR/help.out" +rg -q "G12-settlement_quota_path_evidence.json" "$TMP_DIR/help.out" + +env -u CARGO_TARGET_DIR bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence" >"$TMP_DIR/dry-run.out" + +rg -q "tests=g04 g12" "$TMP_DIR/dry-run.out" +rg -q "rustfs-scanner scanner_recovery_intent" "$TMP_DIR/dry-run.out" +rg -q "e2e_test distributed::replication_quota_test::four_node_four_drive_hard_quota_rejects_over_limit_put" "$TMP_DIR/dry-run.out" +rg -q "target_dir=$PROJECT_ROOT/target" "$TMP_DIR/dry-run.out" + +RUSTFS_SCANNER_HEAL_W16_OUTPUT_ROOT="$TMP_DIR/root-out" \ + bash "$RUNNER" --dry-run >"$TMP_DIR/dry-run-output-root.out" +rg -q "run_dir=$TMP_DIR/root-out/" "$TMP_DIR/dry-run-output-root.out" + +CARGO_TARGET_DIR="$TMP_DIR/shared-target" bash "$RUNNER" \ + --dry-run \ + --out-dir "$TMP_DIR/evidence-with-target" >"$TMP_DIR/dry-run-target.out" +rg -q "target_dir=$TMP_DIR/shared-target" "$TMP_DIR/dry-run-target.out" +rg -q "current_binary=$TMP_DIR/shared-target/debug/rustfs" "$TMP_DIR/dry-run-target.out" + +mkdir -p "$TMP_DIR/nonempty/g04-crash-boundaries" +touch "$TMP_DIR/nonempty/g04-crash-boundaries/existing.json" +if bash "$RUNNER" --dry-run --out-dir "$TMP_DIR/nonempty" >"$TMP_DIR/nonempty.out" 2>"$TMP_DIR/nonempty.err"; then + echo "W16 runner should reject non-empty evidence case directories" >&2 + exit 1 +fi +rg -q "evidence case directory is not empty" "$TMP_DIR/nonempty.err" + +bash "$RUNNER" --self-test diff --git a/scripts/test_summarize_scanner_heal_perf.py b/scripts/test_summarize_scanner_heal_perf.py index ca5c084de..f59b4b46c 100755 --- a/scripts/test_summarize_scanner_heal_perf.py +++ b/scripts/test_summarize_scanner_heal_perf.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import contextlib import hashlib import io @@ -15,6 +16,7 @@ from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parent)) import summarize_scanner_heal_perf as summary +import check_test_wiring as wiring def sha(path: Path) -> str: @@ -33,6 +35,10 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): self.abba = self.root / "abba" self.abba.mkdir() self.manifest = { + "schema": 1, + "evidence": "measured", + "rounds": 3, + "duration_seconds": summary.MIN_MEASURED_RELEASE_DURATION_SECONDS, "fixed": { "config_sha256": "1" * 64, "dataset_sha256": "2" * 64, @@ -45,26 +51,151 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): "candidate": {"revision": "b" * 40, "sha256": "4" * 64}, "adapter_sha256": "5" * 64, "collector_sha256": "6" * 64, + "release_evidence": { + "topology": { + "nodes": 3, + "drives_per_node": 4, + "pools": 2, + "sets_total": 2, + "sampled_pools": 2, + "sampled_sets": 2, + "erasure_set_size": 12, + "erasure_data_blocks": 8, + "erasure_parity_blocks": 4, + }, + "distributed": { + "metrics_endpoints": ["https://node-1:9000", "https://node-2:9000", "https://node-3:9000"], + "failure_domain": "three-node-localhost-lab", + "same_window_sampling": True, + }, + "scheduler": { + "bounds": ["admission-retry-idempotency", "deadline-budget", "lock-hold-bound", "minimum-progress"], + "max_deferred_items": 128, + "max_deferred_bytes": 1048576, + "max_retry_age_seconds": 300, + "duplicate_task_bound_observed": True, + }, + "crash_restart": { + "fault_modes": ["process-restart", "process-crash-restart"], + "unclean_shutdown_marker": True, + }, + "mixed_version": { + "participating_revisions": ["a" * 40, "b" * 40], + "reader": True, + "writer": True, + "rollback_payload": True, + }, + "profile": { + "required_artifacts": ["allocation-profile", "flamegraph", "rss-samples", "save-frequency"], + "collector_config_sha256": "7" * 64, + "profiler_config_sha256": "8" * 64, + "measurements": { + "resolved_samples": 120, + "allocation_bytes": 4096, + "rss_peak_bytes": 10485760, + "save_operations": 64, + "saved_bytes": 8192, + }, + }, + "heal_capacity": { + "objects": 96, + "versions": 96, + "bytes": 12582912, + "completed_objects": 96, + }, + "recovery_window": { + "pressure_recovery_window_seconds": 45, + "heal_lock_wait_p99_ms": 8, + "recovery_p95_ms": 1500, + "recovery_p99_ms": 2200, + }, + }, } self.comparison = { "scenario": "cold-hot", "comparison": "build", "round": 1, "status": "pass", + "foreground_p95_ms": 8.0, + "foreground_p99_ms": 10.0, + "throughput_ops": 100.0, + "error_rate": 0.0, "p99_regression": 0.02, "throughput_change": -0.01, - "p1": {"required_reduction": 0.8, "observed_reduction": 0.82, "repeatability_drift": 0.01}, + "p1": { + "required_reduction": 0.8, + "observed_reduction": 0.82, + "repeatability_drift": 0.01, + "baseline_walk_objects": 100, + "baseline_cold_walk_objects": 100, + "candidate_walk_objects": 20, + "candidate_cold_walk_objects": 0, + }, "p2_post_stop_work_multiples": [None, 1.1, 1.0, None], + "w10_w11": { + "foreground_pressure_samples": [10, 10, 10, 10], + "foreground_pressure_high_samples": [0, 3, 3, 0], + "foreground_pressure_high_sample_ratios": [0.0, 0.25, 0.25, 0.0], + "heal_lock_wait_p99_ms": [12.0, 8.0, 9.0, 13.0], + "attempt_cost_per_healed_object": [None, 1.2, 1.3, None], + "candidate_attempt_cost_per_healed_object": 1.3, + }, + "w09": { + "heal_start_p95_ms": [42.0, 40.0, 41.0, 43.0], + "heal_duplicate_task_count": [0, 0, 0, 0], + "heal_lock_hold_p95_ms": [7.0, 6.0, 6.5, 7.5], + }, + "w10": { + "status": "not_applicable", + "pacing_observed": False, + }, + "w11": {"status": "not_applicable"}, } self.report = { "status": "pass", "performance": "pass", "evidence": "measured", "cells": 120, - "comparisons": [self.comparison], + "comparisons": self.full_comparisons(), + "started_at": "2026-09-09T00:00:00Z", + "finished_at": "2026-09-09T02:30:00Z", } self.write_inputs() + def full_comparisons(self): + comparisons = [] + for scenario in summary.SCENARIOS: + for comparison in ("build", "background"): + for round_id in range(1, 4): + row = copy.deepcopy(self.comparison) + row.update(scenario=scenario, comparison=comparison, round=round_id) + if scenario == "running-heal" and comparison == "build": + row["w10"] = { + "status": "observed", + "pacing_observed": True, + "candidate_pressure_high_ratio": 0.3, + "candidate_delay_events": 3, + "foreground_p99_change": -0.02, + "foreground_throughput_change": 0.01, + } + row["w11"] = { + "status": "observed", + "rss_growth_limit": 0.05, + "rss_growth": 0.01, + "rss_within_limit": True, + "baseline_rss_bytes": 1000000.0, + "candidate_rss_bytes": 1010000.0, + "baseline_heal_lock_wait_p99_ms": 12.0, + "candidate_heal_lock_wait_p99_ms": 8.0, + "heal_lock_wait_p99_change": -0.33, + "healthy_page_latency_observed": True, + "foreground_p99_change": -0.02, + "foreground_throughput_change": 0.01, + "candidate_attempt_cost_per_healed_object": 1.3, + } + comparisons.append(row) + return comparisons + def write_inputs(self): (self.abba / "manifest.json").write_text(json.dumps(self.manifest), encoding="utf-8") (self.abba / "report.json").write_text(json.dumps(self.report), encoding="utf-8") @@ -96,8 +227,71 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): result = summary.build_summary(args) self.assertEqual(result["verdict"], "PASS") self.assertEqual(result["abba"]["provenance"]["manifest_sha256"], sha(self.abba / "manifest.json")) + self.assertEqual(result["abba"]["w09_duplicate_task_count"], 0.0) + self.assertEqual(result["abba"]["w09_worst_heal_start_p95_ms"], 43.0) + self.assertEqual( + [row["status"] for row in result["abba"]["w11_running_heal_build"]], + ["observed", "observed", "observed"], + ) + self.assertIn("w11_running_heal_build_statuses: observed,observed,observed", summary.markdown(result)) self.assertEqual(result["cache_cost"]["max_save_body_amplification"], 2.0) + def test_release_descriptor_binds_g10_p1_p3_measured_artifacts(self): + profile_paths = [] + for kind in summary.RELEASE_PROFILE_ARTIFACTS: + artifact = self.root / f"{kind}.artifact" + artifact.write_text(f"{kind} measured profile\n", encoding="utf-8") + profile_paths.append(f"{kind}={artifact}") + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "release_bundle_descriptor_out": self.root / "release-descriptor.json", + "release_source_revision": "b" * 40, + "release_profile_artifact": profile_paths, + }) + result = summary.build_summary(args) + summary.write_release_bundle_descriptor(args, result) + descriptor = summary.read_json(args.release_bundle_descriptor_out) + self.assertEqual(sorted(descriptor["gates"]), ["G10", "P1", "P3"]) + self.assertEqual( + descriptor["gates"]["G10"]["evidence_fields"]["scheduler_bound_evidence"]["scheduler_bounds"], + list(summary.RELEASE_SCHEDULER_BOUNDS), + ) + profile = descriptor["gates"]["P1"]["evidence_fields"]["profile_evidence"] + self.assertEqual(sorted(profile["profile_artifacts"]), sorted(summary.RELEASE_PROFILE_ARTIFACTS)) + self.assertEqual( + profile["measurement_window_id"], + descriptor["gates"]["P3"]["evidence_fields"]["two_hour_pressure_measurement"]["measurement_window_id"], + ) + for gate in ("G10", "P1", "P3"): + with mock.patch("subprocess.check_output", return_value="b" * 40): + status = wiring.scanner_heal_release_bundle_gate_status( + Path(__file__).resolve().parents[1], + args.release_bundle_descriptor_out, + gate, + ) + self.assertEqual(status["verified_gate"], gate) + save_frequency = profile["profile_artifacts"]["save-frequency"] + wrapper = args.release_bundle_descriptor_out.parent / save_frequency["artifact"] + payload = summary.read_json(wrapper) + raw_profile = args.release_bundle_descriptor_out.parent / payload["raw_profile_artifact"] + self.assertTrue(raw_profile.is_file()) + self.assertEqual(payload["saved_bytes"], 8192) + self.assertEqual(payload["raw_profile_sha256"], sha(raw_profile)) + + def test_release_descriptor_requires_profile_artifacts(self): + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "release_bundle_descriptor_out": self.root / "release-descriptor.json", + "release_source_revision": "b" * 40, + "release_profile_artifact": [], + }) + with self.assertRaisesRegex(ValueError, "missing profile artifacts"): + summary.write_release_bundle_descriptor(args, summary.build_summary(args)) + def test_synthetic_report_fails_as_performance_conclusion(self): self.report.update(status="synthetic_validated", performance="pending", evidence="synthetic") self.write_inputs() @@ -112,6 +306,158 @@ class ScannerHealPerfSummaryTest(unittest.TestCase): self.assertEqual(result["verdict"], "FAIL") self.assertIn("synthetic evidence", result["reason"]) + def test_failed_abba_report_without_comparisons_writes_fail_closed_summary(self): + self.report = { + "status": "failed", + "performance": "pending", + "completed_cells": 7, + "error": "collector failed", + } + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + result = summary.build_summary(args) + self.assertEqual(result["verdict"], "FAIL") + self.assertEqual(result["abba"]["completed_cells"], 7) + self.assertEqual(result["abba"]["comparisons_total"], 0) + self.assertIn("collector failed", result["reason"]) + self.assertIn("- completed_cells: 7", summary.markdown(result)) + self.assertIn("- error: collector failed", summary.markdown(result)) + + def test_passing_abba_report_requires_comparisons(self): + del self.report["comparisons"] + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "passing report requires comparisons"): + summary.build_summary(args) + + def test_passing_abba_report_requires_complete_matrix(self): + cases = { + "trimmed": lambda: self.report["comparisons"].pop(), + "duplicate": lambda: self.report["comparisons"].__setitem__(1, copy.deepcopy(self.report["comparisons"][0])), + "bad cells": lambda: self.report.update(cells=119), + "bad evidence": lambda: self.manifest.update(evidence="synthetic"), + "outside": lambda: self.report["comparisons"][0].update(round=99), + "failed comparison": lambda: self.report["comparisons"][0].update(status="inconclusive"), + } + for name, mutate in cases.items(): + with self.subTest(fault=name): + self.setUp() + mutate() + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "ABBA matrix|manifest/report evidence|comparison"): + summary.build_summary(args) + + def test_passing_measured_report_requires_two_hour_window(self): + self.manifest["duration_seconds"] = summary.MIN_MEASURED_RELEASE_DURATION_SECONDS - 1 + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "two hours"): + summary.build_summary(args) + + def test_passing_abba_report_requires_w10_w11_evidence(self): + for fault in ("missing", "pressure", "lock", "attempt", "length", "range", "w11-missing", "w11-pending"): + with self.subTest(fault=fault): + self.setUp() + target = self.report["comparisons"][0] + if fault == "missing": + del target["w10_w11"] + elif fault == "pressure": + del target["w10_w11"]["foreground_pressure_high_sample_ratios"] + elif fault == "lock": + del target["w10_w11"]["heal_lock_wait_p99_ms"] + elif fault == "attempt": + del target["w10_w11"]["attempt_cost_per_healed_object"] + elif fault == "length": + target["w10_w11"]["attempt_cost_per_healed_object"] = [None] + elif fault == "w11-missing": + running_heal = next( + comparison for comparison in self.report["comparisons"] + if comparison["scenario"] == "running-heal" and comparison["comparison"] == "build" + ) + del running_heal["w11"] + elif fault == "w11-pending": + running_heal = next( + comparison for comparison in self.report["comparisons"] + if comparison["scenario"] == "running-heal" and comparison["comparison"] == "build" + ) + running_heal["w11"]["status"] = "pending" + else: + target["w10_w11"]["foreground_pressure_high_sample_ratios"] = [1.5, 0.0, 0.0, 0.0] + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "W10/W11|W11|performance evidence|length mismatch|above maximum"): + summary.build_summary(args) + + def test_passing_abba_report_requires_w09_evidence(self): + cases = { + "missing": lambda row: row.pop("w09"), + "start": lambda row: row["w09"].pop("heal_start_p95_ms"), + "zero start": lambda row: row["w09"].update(heal_start_p95_ms=[0, 40.0, 41.0, 43.0]), + "duplicates": lambda row: row["w09"].update(heal_duplicate_task_count=[0, 1, 0, 0]), + "unknown duplicates": lambda row: row["w09"].update(heal_duplicate_task_count=[None, 0, 0, 0]), + "lock": lambda row: row["w09"].pop("heal_lock_hold_p95_ms"), + "zero lock": lambda row: row["w09"].update(heal_lock_hold_p95_ms=[0, 6.0, 6.5, 7.5]), + "length": lambda row: row["w09"].update(heal_lock_hold_p95_ms=[1]), + } + for name, mutate in cases.items(): + with self.subTest(fault=name): + self.setUp() + mutate(self.report["comparisons"][0]) + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "W09|performance evidence|above maximum|length mismatch|must be measured"): + summary.build_summary(args) + + def test_passing_measured_report_requires_release_evidence_manifest(self): + del self.manifest["release_evidence"] + self.write_inputs() + args = type("Args", (), { + "abba_dir": self.abba, + "cache_cost_log": None, + "require_cache_cost": False, + "json_out": None, + "markdown_out": None, + }) + with self.assertRaisesRegex(ValueError, "release_evidence"): + summary.build_summary(args) + def test_requires_cache_profile_when_requested(self): args = type("Args", (), { "abba_dir": self.abba,