opensource/rustfs
3
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 19:24:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(common): move scanner/heal contracts into dedicated crates (#6615)

Browse Source 
refactor(common): move scanner/heal domain contracts into dedicated crates

crates/common carried ~5.6K lines of scanner/heal domain code (metrics.rs,
heal_channel.rs, last_minute.rs) parked there to break dependency cycles;
every scanner type change recompiled all 11 rustfs-common dependents.

Pure move, zero renames, zero shape changes (backlog#1843):

- New crate rustfs-heal-contracts receives heal_channel.
- New crate rustfs-scanner-contracts receives metrics, last_minute, and the
  GLOBAL_INIT_TIME trio (metrics::report() reads it as the current-cycle
  fallback, so it must live below the shim to avoid a dependency cycle).
- rustfs-common re-exports everything at the old paths as a transitional
  shim; consumers migrate crate by crate, then the shims are deleted.
This commit is contained in:
Zhengchao An
2026-08-26 10:49:12 +08:00
committed by GitHub
parent 37a50f1e5d
commit df5ae13fd0
14 changed files with 204 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ARCHITECTURE.md
+2 -2
View File
@@ -92,7 +92,7 @@ refactors.
| Domain | Current workspace crates | Responsibility |
|--------|--------------------------|----------------|
| Foundation | `checksums`, `common`, `config`, `data-usage`, `utils` | Shared configuration, data-usage models, utilities, and checksums. |
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-contracts`, `utils` | Shared configuration, data-usage models, heal/scanner domain contracts, utilities, and checksums. |
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, and I/O pipelines. |
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
@@ -128,7 +128,7 @@ default build (lifecycle:
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
collision, not copies; renaming is tracked in rustfs/backlog#1847.
- `LastMinuteLatency` has two deliberately different implementations: the
per-second bucketed accumulator in `crates/common/src/last_minute.rs` and
per-second bucketed accumulator in `crates/scanner-contracts/src/last_minute.rs` and
the in-memory endpoint-health sample tracker in
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
why it stays local).
Cargo.lock
Generated
+26 -5
View File
@@ -9390,13 +9390,10 @@ dependencies = [
name = "rustfs-common"
version = "1.0.0-rc.3"
dependencies = [
"chrono",
"hotpath",
"jiff",
"metrics",
"rmp-serde",
"serde",
"serde_json",
"rustfs-heal-contracts",
"rustfs-scanner-contracts",
"smallvec",
"tokio",
"tonic",
@@ -9684,6 +9681,15 @@ dependencies = [
"walkdir",
]
[[package]]
name = "rustfs-heal-contracts"
version = "1.0.0-rc.3"
dependencies = [
"serde",
"tokio",
"uuid",
]
[[package]]
name = "rustfs-iam"
version = "1.0.0-rc.3"
@@ -10416,6 +10422,21 @@ dependencies = [
"uuid",
]
[[package]]
name = "rustfs-scanner-contracts"
version = "1.0.0-rc.3"
dependencies = [
"chrono",
"jiff",
"metrics",
"rmp-serde",
"rustfs-heal-contracts",
"serde",
"serde_json",
"tokio",
"uuid",
]
[[package]]
name = "rustfs-security-governance"
version = "1.0.0-rc.3"
Cargo.toml
+4
View File
@@ -26,6 +26,7 @@ members = [
"crates/e2e_test", # End-to-end test suite
"crates/filemeta", # File metadata management
"crates/heal", # Erasure set and object healing
"crates/heal-contracts", # Heal request/response channel contracts
"crates/iam", # Identity and Access Management
"crates/keystone", # OpenStack Keystone integration
"crates/lifecycle", # Lifecycle rule evaluation contracts
@@ -49,6 +50,7 @@ members = [
"crates/s3select-api", # S3 Select API interface
"crates/s3select-query", # S3 Select query engine
"crates/scanner", # Scanner for data integrity checks and health monitoring
"crates/scanner-contracts", # Scanner metrics and cycle contracts
"crates/security-governance", # Security governance contracts
"crates/extension-schema", # Extension schema contracts
"crates/signer", # client signer
@@ -88,6 +90,8 @@ redundant_clone = "warn"
# RustFS Internal Crates
rustfs = { path = "./rustfs", version = "1.0.0-rc.3" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.3" }
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.3" }
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.3" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.3" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.3" }
rustfs-common = { path = "crates/common", version = "1.0.0-rc.3" }
crates/common/Cargo.toml
+4 -7
View File
@@ -38,16 +38,13 @@ hotpath.workspace = true
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
tonic = { workspace = true, features = ["gzip", "deflate"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
chrono = { workspace = true, features = ["serde"] }
jiff = { workspace = true, features = ["serde"] }
metrics = { workspace = true }
serde = { workspace = true, features = ["derive"] }
# Transitional shim deps (backlog#1843): dropped with the re-export shims once
# every consumer imports the contracts crates directly.
rustfs-heal-contracts = { workspace = true }
rustfs-scanner-contracts = { workspace = true }
smallvec = { workspace = true }
rmp-serde = { workspace = true }
tracing = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
[lib]
doctest = false
crates/common/src/globals.rs
+4 -17
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use chrono::{DateTime, Utc};
use std::collections::HashMap;
use std::sync::LazyLock;
use std::sync::atomic::{AtomicU64, Ordering};
@@ -27,8 +26,10 @@ pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLoc
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
/// Global initialization time of the RustFS node.
pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<DateTime<Utc>>>> = LazyLock::new(|| RwLock::new(None));
// Transitional re-export shim (backlog#1843): the node init-time global moved
// to rustfs-scanner-contracts, whose metrics report reads it directly.
pub use rustfs_scanner_contracts::{GLOBAL_INIT_TIME, get_global_init_time, set_global_init_time_now};
/// Log level to use when reporting cached gRPC connection eviction.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -63,20 +64,6 @@ pub fn try_get_global_local_node_name() -> Option<String> {
.filter(|name| !name.is_empty())
}
/// Set the global RustFS initialization time to the current UTC time.
pub async fn set_global_init_time_now() {
let now = Utc::now();
*GLOBAL_INIT_TIME.write().await = Some(now);
}
/// Get the global RustFS initialization time.
///
/// # Returns
/// * `Option<DateTime<Utc>>` - The initialization time if set.
pub async fn get_global_init_time() -> Option<DateTime<Utc>> {
*GLOBAL_INIT_TIME.read().await
}
/// Set the global RustFS address used for gRPC connections.
///
/// # Arguments
crates/common/src/lib.rs
+7 -3
View File
@@ -14,14 +14,18 @@
// pub mod error;
pub mod globals;
pub mod heal_channel;
pub mod last_minute;
pub mod metrics;
pub mod mrf_channel;
mod readiness;
pub mod table_catalog;
pub mod trace_bus;
// Transitional re-export shims (backlog#1843): these modules moved to the
// rustfs-heal-contracts / rustfs-scanner-contracts crates. Consumers migrate
// to the new paths crate by crate; the shims are deleted once
// `rg 'rustfs_common::(metrics|heal_channel|last_minute)'` reports zero hits.
pub use rustfs_heal_contracts::heal_channel;
pub use rustfs_scanner_contracts::{last_minute, metrics};
pub use globals::*;
pub use readiness::{GlobalReadiness, SystemStage};
crates/heal-contracts/Cargo.toml
+39
View File
@@ -0,0 +1,39 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-heal-contracts"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "Heal request/response channel contracts shared by heal producers and the heal executor."
keywords = ["heal", "contracts", "channel", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "data-structures"]
[lints]
workspace = true
[dependencies]
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["sync"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
[dev-dependencies]
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
[lib]
doctest = false
crates/common/src/heal_channel.rs → crates/heal-contracts/src/heal_channel.rs
View File
crates/heal-contracts/src/lib.rs
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod heal_channel;
crates/scanner-contracts/Cargo.toml
+45
View File
@@ -0,0 +1,45 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-scanner-contracts"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "Scanner metrics and lifecycle-cycle contracts shared by the scanner, storage engine, and observability layers."
keywords = ["scanner", "contracts", "metrics", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "data-structures"]
[lints]
workspace = true
[dependencies]
chrono = { workspace = true, features = ["serde"] }
jiff = { workspace = true, features = ["serde"] }
metrics = { workspace = true }
rmp-serde = { workspace = true }
rustfs-heal-contracts = { workspace = true }
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["sync"] }
[dev-dependencies]
serde_json = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
uuid = { workspace = true, features = ["v4"] }
[lib]
doctest = false
crates/scanner-contracts/src/init_time.rs
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use chrono::{DateTime, Utc};
use std::sync::LazyLock;
use tokio::sync::RwLock;
/// Global initialization time of the RustFS node.
pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<DateTime<Utc>>>> = LazyLock::new(|| RwLock::new(None));
/// Set the global RustFS initialization time to the current UTC time.
pub async fn set_global_init_time_now() {
let now = Utc::now();
*GLOBAL_INIT_TIME.write().await = Some(now);
}
/// Get the global RustFS initialization time.
///
/// # Returns
/// * `Option<DateTime<Utc>>` - The initialization time if set.
pub async fn get_global_init_time() -> Option<DateTime<Utc>> {
*GLOBAL_INIT_TIME.read().await
}
crates/common/src/last_minute.rs → crates/scanner-contracts/src/last_minute.rs
View File
crates/scanner-contracts/src/lib.rs
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod init_time;
pub mod last_minute;
pub mod metrics;
pub use init_time::{GLOBAL_INIT_TIME, get_global_init_time, set_global_init_time_now};
crates/common/src/metrics.rs → crates/scanner-contracts/src/metrics.rs
+5 -5
View File
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::heal_channel::HealScanMode;
use crate::last_minute::{AccElem, LastMinuteLatency};
use chrono::{DateTime, Utc};
use jiff::Timestamp;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use serde::{Deserialize, Serialize};
use std::{
collections::{BTreeSet, HashMap},
@@ -3243,7 +3243,7 @@ impl Metrics {
has_cycle
};
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
if !has_cycle && let Some(init_time) = crate::init_time::get_global_init_time().await {
m.current_started = chrono_to_jiff_timestamp(init_time);
}
@@ -4401,10 +4401,10 @@ mod tests {
#[tokio::test]
async fn report_preserves_current_cycle_started_time() {
let previous_init_time = *crate::globals::GLOBAL_INIT_TIME.read().await;
let previous_init_time = *crate::init_time::GLOBAL_INIT_TIME.read().await;
let init_time = Utc::now() - chrono::Duration::hours(1);
let cycle_started = Utc::now();
*crate::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
*crate::init_time::GLOBAL_INIT_TIME.write().await = Some(init_time);
let metrics = Metrics::new();
metrics
@@ -4416,7 +4416,7 @@ mod tests {
.await;
let report = metrics.report().await;
*crate::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
*crate::init_time::GLOBAL_INIT_TIME.write().await = previous_init_time;
assert_eq!(report.current_started, chrono_to_jiff_timestamp(cycle_started));
}