opensource/rustfs
3
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-10-13 09:17:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into improve/base

Browse Source
This commit is contained in:
Hauser
2026-10-09 16:26:00 +08:00
committed by GitHub
parent f4ce1d8826 25e73d3bd6
commit f1f58c9619
8 changed files with 79 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/nextest.toml
+10
View File
@@ -92,6 +92,12 @@ setup = 'ecstore-large-stack'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
# These handler/store fixtures and the relocated-pool GET compose deep storage
# futures in libtest threads; on Linux they overflow the default 2 MiB stack.
[[profile.default.scripts]]
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
setup = 'ecstore-base-stack'
[[profile.default.scripts]]
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
setup = 'lifecycle-large-stack'
@@ -337,6 +343,10 @@ setup = 'ecstore-large-stack'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs) & (test(/^admin::handlers::table_catalog::tests::/) | test(/^table_catalog::store::tests::/) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
setup = 'lifecycle-large-stack'
.github/workflows/build.yml
+11
View File
@@ -440,6 +440,17 @@ jobs:
fi
done
- name: Verify macOS binary has no dynamic liblzma dependency
if: matrix.platform == 'macos'
shell: bash
run: |
binary="target/${{ matrix.target }}/release/rustfs"
if otool -L "$binary" | grep -E 'liblzma.*\.dylib'; then
echo "macOS release binary must not depend on a dynamic liblzma library" >&2
exit 1
fi
"$binary" --version
- name: Generate the CPU symbol catalog
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
shell: bash
Cargo.lock
Generated
+1
View File
@@ -11062,6 +11062,7 @@ dependencies = [
"async-compression",
"futures",
"hotpath",
"liblzma",
"rustfs-rio",
"rustfs-tokio-tar",
"serde",
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+29 -6
View File
@@ -2939,6 +2939,20 @@ fn resolve_tier_free_version_recovery_enabled(value: Result<String, env::VarErro
}
fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<()>) -> Option<JoinHandle<()>> {
spawn_tier_free_version_recovery_once_with_options(
api,
started,
TIER_FREE_VERSION_RECOVERY_LEADER_LOCK.to_string(),
tier_free_version_recovery_leader_retry_delay,
)
}
fn spawn_tier_free_version_recovery_once_with_options(
api: Arc<ECStore>,
started: &OnceLock<()>,
leader_lock_name: String,
retry_delay: fn() -> StdDuration,
) -> Option<JoinHandle<()>> {
if !tier_free_version_recovery_enabled() {
warn!(
event = EVENT_LIFECYCLE_WORKER_STATE,
@@ -2956,15 +2970,15 @@ fn spawn_tier_free_version_recovery_once(api: Arc<ECStore>, started: &OnceLock<(
Some(tokio::spawn(async move {
let cancel_token = api.ctx.background_cancel_token().unwrap_or_default();
let acquire_api = Arc::clone(&api);
let acquire_lock_name = Arc::<str>::from(leader_lock_name);
run_tier_free_version_recovery_coordinator(
cancel_token,
tier_free_version_recovery_leader_retry_delay,
retry_delay,
move || {
let api = Arc::clone(&acquire_api);
let lock_name = Arc::clone(&acquire_lock_name);
async move {
let lock = api
.new_ns_lock(RUSTFS_META_BUCKET, TIER_FREE_VERSION_RECOVERY_LEADER_LOCK)
.await?;
let lock = api.new_ns_lock(RUSTFS_META_BUCKET, &lock_name).await?;
match lock.get_write_lock_quiet(TIER_FREE_VERSION_RECOVERY_LOCK_TIMEOUT).await {
Ok(guard) => Ok(Some(guard)),
Err(LockError::Timeout { .. }) => Ok(None),
@@ -14193,6 +14207,10 @@ mod tests {
.await;
}
fn fast_tier_free_version_recovery_leader_retry_delay() -> StdDuration {
StdDuration::from_millis(10)
}
#[tokio::test]
#[serial]
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
@@ -14208,8 +14226,13 @@ mod tests {
seed_recoverable_free_version(&disk_paths, &bucket, object, None, None).await;
let started = OnceLock::new();
let recovery = super::spawn_tier_free_version_recovery_once(Arc::clone(&ecstore), &started)
.expect("production recovery entrypoint should start once");
let recovery = super::spawn_tier_free_version_recovery_once_with_options(
Arc::clone(&ecstore),
&started,
format!("tier-free-version-recovery/test/{}", Uuid::new_v4()),
fast_tier_free_version_recovery_leader_retry_delay,
)
.expect("production recovery entrypoint should start once");
let task = tokio::time::timeout(StdDuration::from_secs(30), async { recovery_rx.lock().await.recv().await })
.await
.expect("production recovery entrypoint should enqueue the seeded item")
crates/ecstore/src/store/bucket.rs
+19 -6
View File
@@ -901,6 +901,7 @@ impl ECStore {
}
};
let confirmed_missing = existing_bucket_info.is_none();
let mut pending_creation_metadata = None;
if confirmed_missing
&& !opts.no_lock
&& !is_meta_bucketname(bucket)
@@ -921,13 +922,25 @@ impl ECStore {
),
other => other,
})?;
return Err(StorageError::InvalidArgument(
"CreateBucket".to_string(),
"bucket".to_string(),
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
));
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
let metadata_store = metadata_sys::object_store_in(&self.ctx).await?;
let pending_intent = persisted
&& metadata.bucket_incarnation_id == old_incarnation
&& metadata.needs_bucket_creation_commit()
&& !crate::bucket::retirement::is_retired(metadata_store, bucket, old_incarnation).await?;
if pending_intent {
pending_creation_metadata = Some(metadata);
} else {
return Err(StorageError::InvalidArgument(
"CreateBucket".to_string(),
"bucket".to_string(),
format!("orphaned bucket generation {old_incarnation} requires administrator recovery before recreation"),
));
}
}
let existing_metadata = if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
let existing_metadata = if let Some(metadata) = pending_creation_metadata {
Some(metadata)
} else if opts.force_create && !confirmed_missing && !is_meta_bucketname(bucket) {
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
if !persisted {
metadata = BucketMetadata::new(bucket);
crates/ecstore/src/store/init.rs
+1 -1
View File
@@ -18318,7 +18318,7 @@ mod tests {
let bucket = format!("lock-create-intent-retry-{}", uuid::Uuid::new_v4());
let mut intent = crate::bucket::metadata::BucketMetadata::new(&bucket);
intent.lock_enabled = true;
crate::bucket::metadata_sys::set_new_bucket_metadata_in(&ctx, intent)
crate::bucket::metadata_sys::set_new_bucket_metadata_intent_in(&ctx, intent)
.await
.expect("persist pre-visibility lock intent");
assert!(
crates/zip/Cargo.toml
+4
View File
@@ -49,6 +49,10 @@ rustfs-rio.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
thiserror = { workspace = true }
[target.'cfg(target_os = "macos")'.dependencies]
# Keep XZ decoding self-contained in macOS binaries instead of linking Homebrew liblzma.
liblzma = { version = "0.4.8", features = ["static"] }
[dev-dependencies]
rustfs-tokio-tar = { workspace = true }
futures = { workspace = true }
rustfs/tests/connect_perf_drive.rs
+4 -8
View File
@@ -80,11 +80,7 @@ fn request(scratch_root: &Path) -> DrivePerformanceRequest {
async fn real_local_write_read_is_measured_and_scratch_is_removed() {
let _guard = TEST_LOCK.lock().await;
let root = tempfile::tempdir().expect("scratch root");
let mut request = request(root.path());
// This exercises real sync_all and filesystem reads. Keep a generous
// deadline so a slow CI filesystem does not turn the integration test
// into an assertion about runner load.
request.duration = Duration::from_secs(10);
let request = request(root.path());
let measurement = measure_drive(&request, &CancellationToken::new())
.await
.expect("real drive benchmark");
@@ -112,7 +108,7 @@ async fn real_local_write_read_is_measured_and_scratch_is_removed() {
assert_eq!(measurement.target.reason_code, DriveTargetReasonCode::Complete);
assert_eq!(measurement.target.parameters.scratch_bytes, 32_768);
assert_eq!(measurement.target.parameters.block_bytes, 4_096);
assert_eq!(measurement.target.parameters.duration_millis, 10_000);
assert_eq!(measurement.target.parameters.duration_millis, 1_000);
assert_eq!(measurement.target.parameters.concurrency, 1);
assert_eq!(measurement.target.units.bytes, "BYTE");
assert_eq!(measurement.target.units.duration, "MILLISECOND");
@@ -509,7 +505,7 @@ fn production_cli_measures_and_signs_exact_binary_provenance() {
&output,
"019e3ae0-0000-7000-8000-000000000014",
32_768,
Duration::from_secs(10),
Duration::from_secs(1),
true,
)
.output()
@@ -528,7 +524,7 @@ fn production_cli_measures_and_signs_exact_binary_provenance() {
assert_eq!(target["reasonCode"], "COMPLETE");
assert_eq!(target["parameters"]["scratchBytes"], 32_768);
assert_eq!(target["parameters"]["blockBytes"], 4_096);
assert_eq!(target["parameters"]["durationMillis"], 10_000);
assert_eq!(target["parameters"]["durationMillis"], 1_000);
assert_eq!(target["parameters"]["concurrency"], 1);
assert_eq!(target["units"]["bytes"], "BYTE");
assert_eq!(target["units"]["duration"], "MILLISECOND");