mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-13 09:17:41 +08:00
perf(ecstore): instrument put object hot path
Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
Hauser
co-authored by
heihutu
zhi22915
heihutu
zhi22915
parent
aaba1b40a4
commit
30d416fbb4
Generated
+50
-54
@@ -306,7 +306,7 @@ dependencies = [
|
||||
"num-bigint 0.4.8",
|
||||
"ouroboros",
|
||||
"quad-rand",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"regex-lite",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
@@ -750,7 +750,7 @@ dependencies = [
|
||||
"nuid",
|
||||
"pin-project",
|
||||
"portable-atomic",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"regex",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
@@ -1305,9 +1305,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.14.2"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3296253d3a91b3f938a3f2bcce4daebadcb4aa4228153fcec90f9d23532b4484"
|
||||
checksum = "591e0024cdf2a8de8711860f7e1d5cfbca26b800ce13793a9aa2f4f5918e6d95"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -1318,9 +1318,7 @@ dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
@@ -1331,9 +1329,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.16.2"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d881a7b7ad179fd6611680c9de89f716fb00ab40299a9a7b8c6913e8f7511a8"
|
||||
checksum = "3fbf162725183ec0df77a9dc82ce22d9ba0b6ea7f376e8ec4f6b695727c70698"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -1371,17 +1369,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.6.4"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b791f3ac597193fe1d08b82366986eb1f5bc31f2ac6c194c0855276116c76cd"
|
||||
checksum = "16ecf7989e381d0cbb7ca06104140d940bf38d8eb9004bd3a12d23df03e3d363"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"http-body 0.4.6",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"itoa",
|
||||
@@ -4044,7 +4040,7 @@ dependencies = [
|
||||
"minlz",
|
||||
"opentelemetry-proto",
|
||||
"prost 0.14.4",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
@@ -4799,7 +4795,7 @@ dependencies = [
|
||||
"google-cloud-wkt",
|
||||
"http 1.5.0",
|
||||
"pin-project",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.20",
|
||||
@@ -5197,7 +5193,7 @@ dependencies = [
|
||||
"idna",
|
||||
"ipnet",
|
||||
"jni",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"thiserror 2.0.20",
|
||||
"tinyvec",
|
||||
"tokio",
|
||||
@@ -5217,7 +5213,7 @@ dependencies = [
|
||||
"jni",
|
||||
"once_cell",
|
||||
"prefix-trie",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"ring",
|
||||
"thiserror 2.0.20",
|
||||
"tinyvec",
|
||||
@@ -5242,7 +5238,7 @@ dependencies = [
|
||||
"ndk-context",
|
||||
"once_cell",
|
||||
"parking_lot",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"resolv-conf",
|
||||
"smallvec",
|
||||
"system-configuration",
|
||||
@@ -5463,9 +5459,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.27.9"
|
||||
version = "0.27.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
|
||||
checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53"
|
||||
dependencies = [
|
||||
"http 1.5.0",
|
||||
"hyper",
|
||||
@@ -6030,9 +6026,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
|
||||
|
||||
[[package]]
|
||||
name = "lapin"
|
||||
version = "4.11.0"
|
||||
version = "4.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24b99d8cdfe3f6223f9f75e35e7510a8662767bee27c1cf870d503fc3bb2428a"
|
||||
checksum = "7b86b2a46f42a1d565c7ba327e03725708c1ff212baa4c271cec291e265ae542"
|
||||
dependencies = [
|
||||
"amq-protocol",
|
||||
"async-rs",
|
||||
@@ -6209,9 +6205,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.24"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a"
|
||||
checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -6802,7 +6798,7 @@ dependencies = [
|
||||
"lru 0.18.4",
|
||||
"mysql_common",
|
||||
"percent-encoding",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rustls",
|
||||
"serde",
|
||||
"socket2",
|
||||
@@ -7023,7 +7019,7 @@ checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
@@ -7532,7 +7528,7 @@ dependencies = [
|
||||
"hmac 0.12.1",
|
||||
"pkcs12",
|
||||
"pkcs5 0.7.1",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rc2",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
@@ -7627,7 +7623,7 @@ dependencies = [
|
||||
"delegate",
|
||||
"futures",
|
||||
"log",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"sha2 0.11.0",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
@@ -8134,7 +8130,7 @@ dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"md-5 0.11.0",
|
||||
"memchr",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"sha2 0.11.0",
|
||||
"stringprep",
|
||||
]
|
||||
@@ -8717,7 +8713,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"getrandom 0.4.3",
|
||||
"lru-slab",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rand_pcg",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
@@ -8798,9 +8794,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.2"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
|
||||
checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
|
||||
dependencies = [
|
||||
"chacha20",
|
||||
"getrandom 0.4.3",
|
||||
@@ -9378,7 +9374,7 @@ dependencies = [
|
||||
"pkcs5 0.8.1",
|
||||
"pkcs8 0.11.0",
|
||||
"polyval",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rand_core 0.10.1",
|
||||
"rsa 0.10.0-rc.18",
|
||||
"russh-cryptovec",
|
||||
@@ -9563,7 +9559,7 @@ dependencies = [
|
||||
"prost 0.14.4",
|
||||
"pyroscope",
|
||||
"quick-xml",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rcgen",
|
||||
"regex",
|
||||
"reqwest",
|
||||
@@ -9738,7 +9734,7 @@ dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
"hotpath",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
@@ -9756,7 +9752,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"jsonwebtoken",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rsa 0.10.0-rc.18",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9828,7 +9824,7 @@ dependencies = [
|
||||
"path-absolutize",
|
||||
"pin-project-lite",
|
||||
"proptest",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"ratelimit",
|
||||
"rcgen",
|
||||
"reed-solomon-simd",
|
||||
@@ -10015,7 +10011,7 @@ dependencies = [
|
||||
"openidconnect",
|
||||
"pollster",
|
||||
"qrcode-rs",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"rustfs-config",
|
||||
@@ -10089,7 +10085,7 @@ dependencies = [
|
||||
"kafka-protocol",
|
||||
"metrics",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rustls",
|
||||
"sha2 0.11.0",
|
||||
"socket2",
|
||||
@@ -10112,7 +10108,7 @@ dependencies = [
|
||||
"kafka-protocol",
|
||||
"metrics",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rustfs-kafka",
|
||||
"rustls",
|
||||
"sha2 0.11.0",
|
||||
@@ -10175,7 +10171,7 @@ dependencies = [
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"moka",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"rustfs-s3-types",
|
||||
@@ -10239,7 +10235,7 @@ dependencies = [
|
||||
"futures",
|
||||
"hotpath",
|
||||
"parking_lot",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rustfs-io-metrics",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
@@ -10597,7 +10593,7 @@ dependencies = [
|
||||
"minlz",
|
||||
"pin-project-lite",
|
||||
"proptest",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"reqwest",
|
||||
"rustfs-config",
|
||||
"rustfs-io-metrics",
|
||||
@@ -10629,7 +10625,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"minlz",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-rio",
|
||||
"rustfs-utils",
|
||||
@@ -10659,7 +10655,7 @@ dependencies = [
|
||||
"lazy_static",
|
||||
"md-5 0.11.0",
|
||||
"quick-xml",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rustfs-checksums",
|
||||
"rustfs-config",
|
||||
"rustfs-rio",
|
||||
@@ -10771,7 +10767,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"metrics",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"rmp-serde",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
@@ -11259,7 +11255,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.16.0"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -11317,7 +11313,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-multipart"
|
||||
version = "0.17.0-alpha.1"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bytes",
|
||||
@@ -11330,7 +11326,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-rfc2047"
|
||||
version = "0.16.0"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"thiserror 2.0.20",
|
||||
@@ -11339,7 +11335,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-sigv2"
|
||||
version = "0.16.0"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -11352,7 +11348,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "s3s-sigv4"
|
||||
version = "0.16.0"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6#50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=f3da7ca7684d6e93b8e8250f037aadfe3e599134#f3da7ca7684d6e93b8e8250f037aadfe3e599134"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"base64-simd",
|
||||
@@ -12257,9 +12253,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "suppaftp"
|
||||
version = "12.0.0"
|
||||
version = "12.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c6978f1b04fc86f122f767b88a9d1d89e80cfe3dc97d670598507d713c4ce48"
|
||||
checksum = "ba93b140f4d07f1b3f3fb4d7264b2d422d5c8b4e925eb6a2646e1342d5d2070a"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -12727,7 +12723,7 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
"postgres-protocol",
|
||||
"postgres-types",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -13319,7 +13315,7 @@ checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"
|
||||
dependencies = [
|
||||
"getrandom 0.4.3",
|
||||
"js-sys",
|
||||
"rand 0.10.2",
|
||||
"rand 0.10.3",
|
||||
"serde_core",
|
||||
"sha1_smol",
|
||||
"wasm-bindgen",
|
||||
|
||||
+8
-8
@@ -156,9 +156,9 @@ futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.11.0" }
|
||||
lapin = { default-features = false, version = "4.12.0" }
|
||||
hyper = { version = "1.11.1" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
@@ -249,8 +249,8 @@ aws-sdk-s3 = { default-features = false, version = "1.148.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.116.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.2" }
|
||||
aws-smithy-runtime-api = { version = "1.16.2" }
|
||||
aws-smithy-types = { version = "1.6.4" }
|
||||
aws-smithy-runtime-api = { version = "1.17.0" }
|
||||
aws-smithy-types = { version = "1.8.0" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.0.0"
|
||||
clap = { version = "4.6.7" }
|
||||
@@ -301,7 +301,7 @@ percent-encoding = "2.3.2"
|
||||
qrcode-rs = { version = "2.1.1", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.2" }
|
||||
rand = { version = "0.10.3" }
|
||||
ratelimit = "2.0.0"
|
||||
rayon = "1.12.0"
|
||||
rustfs-erasure-codec = { version = "9.0.0" }
|
||||
@@ -313,8 +313,8 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "50c94aeb1e5ea9ef8e7393ef6bbc87b72f6891c6" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "f3da7ca7684d6e93b8e8250f037aadfe3e599134", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s.git", rev = "f3da7ca7684d6e93b8e8250f037aadfe3e599134" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -364,7 +364,7 @@ pyroscope = { version = "2.1.1" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.0.0" }
|
||||
suppaftp = { version = "12.0.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.63.3" }
|
||||
russh-sftp = "3.0.0"
|
||||
|
||||
@@ -577,7 +577,9 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
SafeNoQuotaCacheFastPathGuard, enable_safe_no_quota_cache_fast_path_for_test, fail_next_quota_ledger_save_for_test,
|
||||
};
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
/// Keep a namespace commit pending until the returned owner is dropped.
|
||||
|
||||
@@ -178,6 +178,13 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
|
||||
Ok(ObjectLockConfigState::ConfirmedAbsent)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct CachedQuotaMetadataSnapshot {
|
||||
pub(crate) quota: Option<BucketQuota>,
|
||||
pub(crate) bucket_incarnation: Uuid,
|
||||
pub(crate) quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Convert the persisted serving-layer configuration into the storage-level
|
||||
/// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention)
|
||||
/// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a
|
||||
@@ -1308,6 +1315,9 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
|
||||
return Err(Error::other("persisted bucket quota configuration is invalid"));
|
||||
}
|
||||
Ok((
|
||||
metadata.quota_config.clone(),
|
||||
metadata.bucket_incarnation_id,
|
||||
@@ -1322,6 +1332,15 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_in(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_cached_quota_config_and_incarnation_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Option<CachedQuotaMetadataSnapshot>> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -2727,6 +2746,27 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result<Option<CachedQuotaMetadataSnapshot>> {
|
||||
let metadata = {
|
||||
let metadata_map = self.metadata_map.read().await;
|
||||
metadata_map.get(bucket).cloned()
|
||||
};
|
||||
let Some(metadata) = metadata else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() {
|
||||
return Ok(None);
|
||||
}
|
||||
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
|
||||
return Err(Error::other("persisted bucket quota configuration is invalid"));
|
||||
}
|
||||
Ok(Some(CachedQuotaMetadataSnapshot {
|
||||
quota: metadata.quota_config.clone(),
|
||||
bucket_incarnation: metadata.bucket_incarnation_id,
|
||||
quota_revision: metadata.quota_config_updated_at,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ use futures::{StreamExt, stream};
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
@@ -41,9 +41,13 @@ const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement";
|
||||
const EVENT_QUOTA_ADMISSION: &str = "quota_admission";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
|
||||
const QUOTA_CONFIG_READ_COUNTERFACTUAL_ENV: &str = "RUSTFS_QUOTA_BEGIN_ASSUME_NO_QUOTA_COUNTERFACTUAL";
|
||||
const SAFE_NO_QUOTA_CACHE_FAST_PATH_ENV: &str = "RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH";
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
static SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
|
||||
// Lock order: caller-held destination object/upload, bucket metadata
|
||||
// transaction (read), operation reservation, then quota ledger.
|
||||
@@ -448,6 +452,11 @@ pub(crate) async fn begin(
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
meta_bucket_fast_path_started,
|
||||
);
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -501,9 +510,105 @@ pub(crate) async fn begin(
|
||||
});
|
||||
}
|
||||
|
||||
let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
|
||||
metadata_lock_started,
|
||||
);
|
||||
let cache_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let cached_quota_snapshot = match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await {
|
||||
Ok(Some(snapshot)) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("candidate", quota_cache_reason(snapshot.quota.as_ref()));
|
||||
Some(snapshot)
|
||||
}
|
||||
Ok(None) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota");
|
||||
None
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
|
||||
cache_lookup_started,
|
||||
);
|
||||
if safe_no_quota_cache_fast_path_enabled()
|
||||
&& let Some(cached) = cached_quota_snapshot.as_ref()
|
||||
&& cached.quota.is_none()
|
||||
&& !cached.bucket_incarnation.is_nil()
|
||||
{
|
||||
let safe_hit_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("safe_fast_path_hit", "no_quota");
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_HIT,
|
||||
safe_hit_started,
|
||||
);
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
ledger_object: ledger_object(bucket),
|
||||
bucket_incarnation: None,
|
||||
quota_revision: None,
|
||||
quota_limit: None,
|
||||
capability_proof: None,
|
||||
snapshot_admission: None,
|
||||
legacy_data_movement: false,
|
||||
metadata_guard: Some(metadata_guard),
|
||||
pool_index: Some(pool_index),
|
||||
set_index: Some(set_index),
|
||||
});
|
||||
}
|
||||
if quota_config_read_counterfactual_enabled() {
|
||||
let counterfactual_skip_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ_COUNTERFACTUAL_SKIP,
|
||||
counterfactual_skip_started,
|
||||
);
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
ledger_object: ledger_object(bucket),
|
||||
bucket_incarnation: None,
|
||||
quota_revision: None,
|
||||
quota_limit: None,
|
||||
capability_proof: None,
|
||||
snapshot_admission: None,
|
||||
legacy_data_movement: false,
|
||||
metadata_guard: Some(metadata_guard),
|
||||
pool_index: Some(pool_index),
|
||||
set_index: Some(set_index),
|
||||
});
|
||||
}
|
||||
let config_read_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (quota, bucket_incarnation, quota_revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?;
|
||||
if let Some(cached) = cached_quota_snapshot.as_ref() {
|
||||
let reason = quota_cache_reason(quota.as_ref());
|
||||
if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason);
|
||||
} else {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason);
|
||||
if cached.quota != quota {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota");
|
||||
}
|
||||
if cached.bucket_incarnation != bucket_incarnation {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation");
|
||||
}
|
||||
if cached.quota_revision != quota_revision {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision");
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
|
||||
config_read_started,
|
||||
);
|
||||
let lock_lost_check_started = rustfs_io_metrics::put_stage_timer();
|
||||
if metadata_guard.is_lock_lost() {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_config",
|
||||
@@ -513,6 +618,11 @@ pub(crate) async fn begin(
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
|
||||
lock_lost_check_started,
|
||||
);
|
||||
let policy_evaluate_started = rustfs_io_metrics::put_stage_timer();
|
||||
if quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
|
||||
@@ -521,6 +631,11 @@ pub(crate) async fn begin(
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
|
||||
policy_evaluate_started,
|
||||
);
|
||||
let capability_proof_started = rustfs_io_metrics::put_stage_timer();
|
||||
let capability_proof = if durable_quota.is_some() {
|
||||
Some(
|
||||
crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
|
||||
@@ -529,7 +644,12 @@ pub(crate) async fn begin(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
|
||||
capability_proof_started,
|
||||
);
|
||||
let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
|
||||
let snapshot_admission_started = rustfs_io_metrics::put_stage_timer();
|
||||
let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) {
|
||||
Some(quota) => match (quota.quota, snapshot_admission) {
|
||||
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
|
||||
@@ -548,6 +668,10 @@ pub(crate) async fn begin(
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
|
||||
snapshot_admission_started,
|
||||
);
|
||||
let legacy_data_movement = durable_quota_limit.is_none()
|
||||
&& quota.as_ref().and_then(|quota| quota.quota).is_some()
|
||||
&& snapshot_admission.is_none()
|
||||
@@ -559,11 +683,21 @@ pub(crate) async fn begin(
|
||||
.then(|| quota.as_ref().and_then(|quota| quota.quota))
|
||||
.flatten()
|
||||
});
|
||||
let object_store_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let store = if durable_quota_limit.is_some() {
|
||||
Some(metadata_sys::object_store_in(ctx).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
|
||||
object_store_lookup_started,
|
||||
);
|
||||
let context_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
|
||||
context_build_started,
|
||||
);
|
||||
Ok(QuotaContext {
|
||||
store,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -581,6 +715,40 @@ pub(crate) async fn begin(
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_config_read_counterfactual_enabled() -> bool {
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(QUOTA_CONFIG_READ_COUNTERFACTUAL_ENV).ok().as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn safe_no_quota_cache_fast_path_enabled() -> bool {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
return true;
|
||||
}
|
||||
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(SAFE_NO_QUOTA_CACHE_FAST_PATH_ENV).ok().as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str {
|
||||
match quota {
|
||||
None => "no_quota",
|
||||
Some(quota) if quota.uses_durable_reservations() => "durable_quota",
|
||||
Some(quota) if quota.quota.is_some() => "snapshot_quota",
|
||||
Some(_) => "quota_without_limit",
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_capability_error(bucket: &str, object: &str) -> StorageError {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
@@ -918,6 +1086,25 @@ pub fn fail_next_quota_ledger_save_for_test() {
|
||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub struct SafeNoQuotaCacheFastPathGuard {
|
||||
previous: bool,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for SafeNoQuotaCacheFastPathGuard {
|
||||
fn drop(&mut self) {
|
||||
SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST.store(self.previous, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[allow(dead_code, reason = "exercised by integration tests behind `--features test-util`")]
|
||||
pub fn enable_safe_no_quota_cache_fast_path_for_test() -> SafeNoQuotaCacheFastPathGuard {
|
||||
let previous = SAFE_NO_QUOTA_CACHE_FAST_PATH_FOR_TEST.swap(true, std::sync::atomic::Ordering::Relaxed);
|
||||
SafeNoQuotaCacheFastPathGuard { previous }
|
||||
}
|
||||
|
||||
fn log_admission_rejected(bucket: &str, object: &str, state: &'static str) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_ADMISSION,
|
||||
|
||||
@@ -42,6 +42,7 @@ use std::{
|
||||
pin::Pin,
|
||||
sync::{LazyLock, Mutex},
|
||||
task::{Context, Poll},
|
||||
time::Instant,
|
||||
};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
use tower::Service;
|
||||
@@ -355,6 +356,41 @@ fn apply_peer_replay_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_data_grpc_stage(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
)
|
||||
}
|
||||
|
||||
async fn observe_put_stage_future<F, T>(
|
||||
future: F,
|
||||
duration_stage: &'static str,
|
||||
pending_count_stage: &'static str,
|
||||
first_pending_to_ready_stage: &'static str,
|
||||
) -> T
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
let duration_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut first_pending_at = None;
|
||||
let mut pending_count = 0usize;
|
||||
let mut future = std::pin::pin!(future);
|
||||
let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) {
|
||||
Poll::Ready(output) => Poll::Ready(output),
|
||||
Poll::Pending => {
|
||||
pending_count = pending_count.saturating_add(1);
|
||||
first_pending_at.get_or_insert_with(Instant::now);
|
||||
Poll::Pending
|
||||
}
|
||||
})
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(duration_stage, duration_started);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(pending_count_stage, pending_count as f64);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at);
|
||||
output
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||
@@ -372,6 +408,12 @@ where
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path());
|
||||
let request_scope_started = if observe_rename_data {
|
||||
rustfs_io_metrics::put_stage_timer()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||
request
|
||||
.headers()
|
||||
@@ -406,11 +448,30 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
|
||||
request_scope_started,
|
||||
);
|
||||
|
||||
let audience = self.audience.clone();
|
||||
let future = self.inner.call(request);
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
let response = if observe_rename_data {
|
||||
observe_put_stage_future(
|
||||
future,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
future.await?
|
||||
};
|
||||
let replay_response_started = if observe_rename_data {
|
||||
rustfs_io_metrics::put_stage_timer()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
|
||||
if let Err(error) = &response_state
|
||||
@@ -428,6 +489,10 @@ where
|
||||
}
|
||||
apply_peer_replay_response(audience, sent_state, response_state);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
|
||||
replay_response_started,
|
||||
);
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -176,6 +176,42 @@ fn attach_mutation_body_digest<T>(
|
||||
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
async fn observe_put_stage_future<F, T>(
|
||||
future: F,
|
||||
duration_stage: &'static str,
|
||||
pending_count_stage: &'static str,
|
||||
first_pending_to_ready_stage: &'static str,
|
||||
) -> T
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
// Keep the normal PUT path identical to the pre-instrumentation future
|
||||
// when detailed stage metrics are disabled. In particular, do not wrap the
|
||||
// RPC in poll_fn: that adds an extra poll layer and pending counter work to
|
||||
// every remote rename even though no samples will be emitted.
|
||||
if !rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
return future.await;
|
||||
}
|
||||
|
||||
let duration_started = rustfs_io_metrics::put_stage_timer();
|
||||
let mut first_pending_at = None;
|
||||
let mut pending_count = 0usize;
|
||||
let mut future = std::pin::pin!(future);
|
||||
let output = std::future::poll_fn(|cx| match future.as_mut().poll(cx) {
|
||||
Poll::Ready(output) => Poll::Ready(output),
|
||||
Poll::Pending => {
|
||||
pending_count = pending_count.saturating_add(1);
|
||||
first_pending_at.get_or_insert_with(Instant::now);
|
||||
Poll::Pending
|
||||
}
|
||||
})
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(duration_stage, duration_started);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(pending_count_stage, pending_count as f64);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(first_pending_to_ready_stage, first_pending_at);
|
||||
output
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
volume_infos
|
||||
.into_iter()
|
||||
@@ -2137,6 +2173,7 @@ impl RemoteDisk {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_data",
|
||||
|| async {
|
||||
let prepare_started = rustfs_io_metrics::put_stage_timer();
|
||||
let file_info = compat_json(fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
@@ -2164,24 +2201,67 @@ impl RemoteDisk {
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
prepare_started,
|
||||
);
|
||||
|
||||
let response = if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await?
|
||||
} else {
|
||||
client.rename_data(request).await?
|
||||
}
|
||||
.into_inner();
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = observe_put_stage_future(
|
||||
async {
|
||||
if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await
|
||||
} else {
|
||||
client.rename_data(request).await
|
||||
}
|
||||
},
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
|
||||
)
|
||||
.await?;
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
|
||||
response_headers_started,
|
||||
);
|
||||
let into_inner_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = response.into_inner();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
|
||||
into_inner_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
|
||||
rpc_started,
|
||||
);
|
||||
|
||||
let success_check_started = rustfs_io_metrics::put_stage_timer();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
|
||||
success_check_started,
|
||||
);
|
||||
|
||||
let decode_started = rustfs_io_metrics::put_stage_timer();
|
||||
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
|
||||
decode_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
|
||||
response_process_started,
|
||||
);
|
||||
|
||||
Ok(rename_data_resp)
|
||||
},
|
||||
@@ -4740,6 +4820,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x14, 0x35]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode");
|
||||
let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test();
|
||||
@@ -5142,6 +5224,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![1_u8; 32]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
|
||||
let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode");
|
||||
|
||||
@@ -47,6 +47,33 @@ use tokio::fs;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn transitioned_old_current_source(
|
||||
dst_meta_existed: bool,
|
||||
dst_meta_unparsable: bool,
|
||||
xlmeta: &FileMeta,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
version_id: Uuid,
|
||||
) -> (bool, Option<FileInfo>) {
|
||||
if dst_meta_unparsable {
|
||||
return (false, None);
|
||||
}
|
||||
if !dst_meta_existed {
|
||||
return (true, None);
|
||||
}
|
||||
let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else {
|
||||
return (false, None);
|
||||
};
|
||||
if old_current.version_id.unwrap_or_default() != version_id {
|
||||
return (true, None);
|
||||
}
|
||||
if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE {
|
||||
(true, Some(old_current))
|
||||
} else {
|
||||
(true, None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Hold later repair publications after admitting one baseline object. The
|
||||
/// fixture arms this on one replacement disk before rejoining the cluster.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
@@ -491,6 +518,14 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let version_id = fi.version_id.unwrap_or_default();
|
||||
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
|
||||
let rollback_data_dir = has_old_data_dir.or_else(|| {
|
||||
@@ -924,6 +959,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})
|
||||
} else {
|
||||
// Inline metadata preparation is blocking. The transaction lease is
|
||||
@@ -937,6 +974,8 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let dst_path_for_failpoint = dst_path.to_string();
|
||||
let dst_volume_for_source = dst_volume.to_string();
|
||||
let dst_path_for_source = dst_path.to_string();
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = rename_commit_guard.clone();
|
||||
@@ -981,6 +1020,14 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
let version_id = fi.version_id.unwrap_or_default();
|
||||
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_for_source,
|
||||
&dst_path_for_source,
|
||||
version_id,
|
||||
);
|
||||
let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
|
||||
let rollback_data_dir = old_data_dir.or_else(|| {
|
||||
@@ -1030,6 +1077,8 @@ impl LocalDisk {
|
||||
old_data_dir,
|
||||
version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
staged_rollback_path,
|
||||
has_dst_buf.is_none(),
|
||||
prepared_metadata_source,
|
||||
@@ -1048,6 +1097,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir,
|
||||
version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
mut local_rollback_path,
|
||||
destination_was_absent,
|
||||
prepared_metadata_source,
|
||||
@@ -1285,6 +1336,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1554,6 +1554,14 @@ pub struct RenameDataResp {
|
||||
/// "cannot vote", never as `Absent`.
|
||||
#[serde(default)]
|
||||
pub old_current_size: Option<OldCurrentSize>,
|
||||
/// Whether this disk decoded enough of the overwritten current version to
|
||||
/// decide if a transitioned source has to be returned.
|
||||
#[serde(default)]
|
||||
pub old_current_source_checked: bool,
|
||||
/// Transitioned old-current source used by PUT free-version cleanup receipt.
|
||||
/// Empty when the old current is not transitioned or no source was known.
|
||||
#[serde(default)]
|
||||
pub old_current_source: Option<FileInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -2001,6 +2009,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(uuid),
|
||||
sign: Some(signature.clone()),
|
||||
old_current_size: Some(OldCurrentSize::Present(42)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
|
||||
assert_eq!(resp.old_data_dir, Some(uuid));
|
||||
@@ -2021,6 +2031,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x01, 0x02, 0x03]),
|
||||
old_current_size,
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
|
||||
let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode");
|
||||
@@ -2058,6 +2070,8 @@ mod tests {
|
||||
assert_eq!(decoded.cleanup_data_dir, None);
|
||||
assert_eq!(decoded.sign, legacy.sign);
|
||||
assert_eq!(decoded.old_current_size, None);
|
||||
assert!(!decoded.old_current_source_checked);
|
||||
assert!(decoded.old_current_source.is_none());
|
||||
}
|
||||
|
||||
/// Test constants
|
||||
|
||||
@@ -382,7 +382,7 @@ async fn flush_read_version_coalescer_pending(
|
||||
fn map_batch_read_version_responses(
|
||||
expected_items: &[ExpectedBatchReadVersionItem],
|
||||
responses: Vec<BatchReadVersionResp>,
|
||||
) -> Vec<crate::disk::error::Result<FileInfo>> {
|
||||
) -> Vec<disk::error::Result<FileInfo>> {
|
||||
let mut results = (0..expected_items.len())
|
||||
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
|
||||
.collect::<Vec<_>>();
|
||||
@@ -3609,6 +3609,9 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) data_dir: Option<Uuid>,
|
||||
pub(in crate::set_disk) cleanup_disks: Vec<Option<DiskStore>>,
|
||||
pub(in crate::set_disk) old_current_size: Option<OldCurrentSize>,
|
||||
pub(in crate::set_disk) old_current_source_checked: bool,
|
||||
pub(in crate::set_disk) old_current_source_all_successful_checked: bool,
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
}
|
||||
@@ -3624,6 +3627,488 @@ pub(in crate::set_disk) struct RenameTailOutcome {
|
||||
pub(in crate::set_disk) cleanup: Vec<RenameTailCleanup>,
|
||||
}
|
||||
|
||||
const RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ENABLE";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS";
|
||||
const RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER";
|
||||
const RENAME_FOREGROUND_ACTIVE_LANES_ENV: &str = "RUSTFS_PUT_RENAME_FOREGROUND_ACTIVE_LANES";
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY: usize = 1024;
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS: usize = 2;
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX: usize = 1;
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW: usize = 128;
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH: usize = 512;
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS: u64 = 500;
|
||||
const DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS: u64 = 2_000;
|
||||
const DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES: usize = 1024;
|
||||
|
||||
fn rename_tail_cleanup_counterfactual_skip_enabled() -> bool {
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV).ok().as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_enabled() -> bool {
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(RENAME_TAIL_CLEANUP_DEFER_ENV).ok().as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_queue_capacity() -> usize {
|
||||
static CAPACITY: OnceLock<usize> = OnceLock::new();
|
||||
*CAPACITY.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY_ENV,
|
||||
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_QUEUE_CAPACITY,
|
||||
)
|
||||
.max(1)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_workers() -> usize {
|
||||
static WORKERS: OnceLock<usize> = OnceLock::new();
|
||||
*WORKERS.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_WORKERS_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_WORKERS).max(1)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_worker_delay() -> Duration {
|
||||
static DELAY: OnceLock<Duration> = OnceLock::new();
|
||||
*DELAY.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY_MS_ENV, 0)))
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_batch_window() -> Duration {
|
||||
static WINDOW: OnceLock<Duration> = OnceLock::new();
|
||||
*WINDOW.get_or_init(|| Duration::from_millis(rustfs_utils::get_env_u64(RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW_MS_ENV, 0)))
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_batch_max() -> usize {
|
||||
static BATCH_MAX: OnceLock<usize> = OnceLock::new();
|
||||
*BATCH_MAX.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX_ENV, DEFAULT_RENAME_TAIL_CLEANUP_DEFER_BATCH_MAX).max(1)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_batch_enabled() -> bool {
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_ENABLE_ENV)
|
||||
.ok()
|
||||
.as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_batch_min() -> usize {
|
||||
static BATCH_MIN: OnceLock<usize> = OnceLock::new();
|
||||
*BATCH_MIN.get_or_init(|| rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_MIN_ENV, 1).max(1))
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_batch_low() -> usize {
|
||||
static BATCH_LOW: OnceLock<usize> = OnceLock::new();
|
||||
*BATCH_LOW.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_LOW_ENV, rename_tail_cleanup_defer_batch_max())
|
||||
.max(1)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_batch_high() -> usize {
|
||||
static BATCH_HIGH: OnceLock<usize> = OnceLock::new();
|
||||
*BATCH_HIGH.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_BATCH_HIGH_ENV,
|
||||
rename_tail_cleanup_defer_adaptive_batch_low().max(rename_tail_cleanup_defer_batch_max()),
|
||||
)
|
||||
.max(1)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_queue_low() -> usize {
|
||||
static QUEUE_LOW: OnceLock<usize> = OnceLock::new();
|
||||
*QUEUE_LOW.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW_ENV,
|
||||
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_LOW,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_queue_high() -> usize {
|
||||
static QUEUE_HIGH: OnceLock<usize> = OnceLock::new();
|
||||
*QUEUE_HIGH.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH_ENV,
|
||||
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_QUEUE_HIGH,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_age_low() -> Duration {
|
||||
static AGE_LOW: OnceLock<Duration> = OnceLock::new();
|
||||
*AGE_LOW.get_or_init(|| {
|
||||
Duration::from_millis(rustfs_utils::get_env_u64(
|
||||
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS_ENV,
|
||||
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_LOW_MS,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_adaptive_age_high() -> Duration {
|
||||
static AGE_HIGH: OnceLock<Duration> = OnceLock::new();
|
||||
*AGE_HIGH.get_or_init(|| {
|
||||
Duration::from_millis(rustfs_utils::get_env_u64(
|
||||
RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS_ENV,
|
||||
DEFAULT_RENAME_TAIL_CLEANUP_DEFER_ADAPTIVE_AGE_HIGH_MS,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_batch_target(queued_depth: usize, oldest_age: Option<Duration>) -> usize {
|
||||
let fixed = rename_tail_cleanup_defer_batch_max();
|
||||
if !rename_tail_cleanup_defer_adaptive_batch_enabled() {
|
||||
return fixed;
|
||||
}
|
||||
|
||||
let min = rename_tail_cleanup_defer_adaptive_batch_min();
|
||||
let low = rename_tail_cleanup_defer_adaptive_batch_low().max(min);
|
||||
let high = rename_tail_cleanup_defer_adaptive_batch_high().max(low);
|
||||
let queue_low = rename_tail_cleanup_defer_adaptive_queue_low();
|
||||
let queue_high = rename_tail_cleanup_defer_adaptive_queue_high().max(queue_low);
|
||||
let age_low = rename_tail_cleanup_defer_adaptive_age_low();
|
||||
let age_high = rename_tail_cleanup_defer_adaptive_age_high().max(age_low);
|
||||
|
||||
let age = oldest_age.unwrap_or_default();
|
||||
if queued_depth >= queue_high || age >= age_high {
|
||||
high
|
||||
} else if queued_depth >= queue_low || age >= age_low {
|
||||
low
|
||||
} else {
|
||||
min
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_defer_hold_worker_enabled() -> bool {
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER_ENV).ok().as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn rename_foreground_active_lanes() -> &'static Vec<AtomicUsize> {
|
||||
static LANES: OnceLock<Vec<AtomicUsize>> = OnceLock::new();
|
||||
LANES.get_or_init(|| {
|
||||
let lanes =
|
||||
rustfs_utils::get_env_usize(RENAME_FOREGROUND_ACTIVE_LANES_ENV, DEFAULT_RENAME_FOREGROUND_ACTIVE_LANES).max(1);
|
||||
(0..lanes).map(|_| AtomicUsize::new(0)).collect()
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn rename_foreground_active_count(disk_index: usize) -> usize {
|
||||
let lanes = rename_foreground_active_lanes();
|
||||
lanes[disk_index % lanes.len()].load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
struct RenameForegroundActiveGuard {
|
||||
disk_index: usize,
|
||||
}
|
||||
|
||||
impl Drop for RenameForegroundActiveGuard {
|
||||
fn drop(&mut self) {
|
||||
let lanes = rename_foreground_active_lanes();
|
||||
lanes[self.disk_index % lanes.len()].fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_foreground_active_guard(disk_index: usize) -> RenameForegroundActiveGuard {
|
||||
let lanes = rename_foreground_active_lanes();
|
||||
let active = lanes[disk_index % lanes.len()].fetch_add(1, Ordering::Relaxed) + 1;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE,
|
||||
active as f64,
|
||||
);
|
||||
RenameForegroundActiveGuard { disk_index }
|
||||
}
|
||||
|
||||
type RenameTailCleanupFuture = Pin<Box<dyn Future<Output = ()> + Send + 'static>>;
|
||||
|
||||
struct RenameTailCleanupJob {
|
||||
enqueued_at: Instant,
|
||||
targets: usize,
|
||||
future: RenameTailCleanupFuture,
|
||||
}
|
||||
|
||||
struct RenameTailCleanupQueue {
|
||||
sender: tokio::sync::mpsc::Sender<RenameTailCleanupJob>,
|
||||
queued: Arc<AtomicUsize>,
|
||||
oldest_enqueued_at: Arc<tokio::sync::Mutex<Option<Instant>>>,
|
||||
}
|
||||
|
||||
async fn run_rename_tail_cleanup_job(job: RenameTailCleanupJob) {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY,
|
||||
job.enqueued_at.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS,
|
||||
job.targets as f64,
|
||||
);
|
||||
let worker_delay = rename_tail_cleanup_defer_worker_delay();
|
||||
if !worker_delay.is_zero() {
|
||||
let delay_started = rustfs_io_metrics::put_stage_timer();
|
||||
tokio::time::sleep(worker_delay).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY,
|
||||
delay_started,
|
||||
);
|
||||
}
|
||||
let cleanup_started = rustfs_io_metrics::put_stage_timer();
|
||||
job.future.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION,
|
||||
cleanup_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED,
|
||||
1.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG,
|
||||
job.enqueued_at.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
|
||||
impl RenameTailCleanupQueue {
|
||||
fn new() -> Self {
|
||||
let (sender, receiver) = tokio::sync::mpsc::channel::<RenameTailCleanupJob>(rename_tail_cleanup_defer_queue_capacity());
|
||||
let queued = Arc::new(AtomicUsize::new(0));
|
||||
let active_workers = Arc::new(AtomicUsize::new(0));
|
||||
let oldest_enqueued_at = Arc::new(tokio::sync::Mutex::new(None::<Instant>));
|
||||
let receiver = Arc::new(tokio::sync::Mutex::new(receiver));
|
||||
if !rename_tail_cleanup_defer_hold_worker_enabled() {
|
||||
for _ in 0..rename_tail_cleanup_defer_workers() {
|
||||
let receiver = Arc::clone(&receiver);
|
||||
let queued = Arc::clone(&queued);
|
||||
let active_workers = Arc::clone(&active_workers);
|
||||
let oldest_enqueued_at = Arc::clone(&oldest_enqueued_at);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let worker_poll_started = rustfs_io_metrics::put_stage_timer();
|
||||
let Some(job) = receiver.lock().await.recv().await else {
|
||||
return;
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT,
|
||||
worker_poll_started,
|
||||
);
|
||||
let mut batch = vec![job];
|
||||
let batch_window = rename_tail_cleanup_defer_batch_window();
|
||||
let queued_snapshot = queued.load(Ordering::Relaxed);
|
||||
let oldest_snapshot = *oldest_enqueued_at.lock().await;
|
||||
let batch_max = rename_tail_cleanup_defer_batch_target(
|
||||
queued_snapshot,
|
||||
oldest_snapshot.map(|oldest| oldest.elapsed()),
|
||||
);
|
||||
if batch_max > 1 && !batch_window.is_zero() {
|
||||
let window_started = rustfs_io_metrics::put_stage_timer();
|
||||
tokio::time::sleep(batch_window).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW,
|
||||
window_started,
|
||||
);
|
||||
}
|
||||
while batch.len() < batch_max {
|
||||
let next = {
|
||||
let mut locked = receiver.lock().await;
|
||||
locked.try_recv().ok()
|
||||
};
|
||||
let Some(next) = next else {
|
||||
break;
|
||||
};
|
||||
batch.push(next);
|
||||
}
|
||||
let queued_after_dequeue = queued.fetch_sub(batch.len(), Ordering::Relaxed).saturating_sub(batch.len());
|
||||
if queued_after_dequeue == 0 {
|
||||
*oldest_enqueued_at.lock().await = None;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
|
||||
queued_after_dequeue as f64,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
queued_after_dequeue as f64,
|
||||
);
|
||||
if let Some(oldest) = *oldest_enqueued_at.lock().await {
|
||||
let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
|
||||
oldest_age_ms,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
|
||||
oldest_age_ms,
|
||||
);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE,
|
||||
batch.len() as f64,
|
||||
);
|
||||
if batch.len() > 1 {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE,
|
||||
batch.len() as f64,
|
||||
);
|
||||
}
|
||||
let active = active_workers.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE,
|
||||
active as f64,
|
||||
);
|
||||
for job in batch {
|
||||
run_rename_tail_cleanup_job(job).await;
|
||||
}
|
||||
active_workers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
Self {
|
||||
sender,
|
||||
queued,
|
||||
oldest_enqueued_at,
|
||||
}
|
||||
}
|
||||
|
||||
async fn try_enqueue(&self, job: RenameTailCleanupJob) -> std::result::Result<(), RenameTailCleanupJob> {
|
||||
let queue_push_started = rustfs_io_metrics::put_stage_timer();
|
||||
{
|
||||
let mut oldest = self.oldest_enqueued_at.lock().await;
|
||||
if oldest.is_none() {
|
||||
*oldest = Some(job.enqueued_at);
|
||||
}
|
||||
}
|
||||
let queued_after_enqueue = self.queued.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if rename_tail_cleanup_defer_hold_worker_enabled() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
|
||||
1.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER,
|
||||
1.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
|
||||
queued_after_enqueue as f64,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
queued_after_enqueue as f64,
|
||||
);
|
||||
if let Some(oldest) = *self.oldest_enqueued_at.lock().await {
|
||||
let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
|
||||
oldest_age_ms,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
|
||||
oldest_age_ms,
|
||||
);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
|
||||
queue_push_started,
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
match self.sender.try_send(job) {
|
||||
Ok(()) => {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
|
||||
1.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
|
||||
queued_after_enqueue as f64,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
queued_after_enqueue as f64,
|
||||
);
|
||||
if let Some(oldest) = *self.oldest_enqueued_at.lock().await {
|
||||
let oldest_age_ms = oldest.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
|
||||
oldest_age_ms,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
|
||||
oldest_age_ms,
|
||||
);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
|
||||
queue_push_started,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => {
|
||||
self.queued.fetch_sub(1, Ordering::Relaxed);
|
||||
if self.queued.load(Ordering::Relaxed) == 0 {
|
||||
*self.oldest_enqueued_at.lock().await = None;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC,
|
||||
1.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
|
||||
self.queued.load(Ordering::Relaxed) as f64,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
self.queued.load(Ordering::Relaxed) as f64,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
|
||||
queue_push_started,
|
||||
);
|
||||
Err(err.into_inner())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_tail_cleanup_queue() -> &'static RenameTailCleanupQueue {
|
||||
static QUEUE: OnceLock<RenameTailCleanupQueue> = OnceLock::new();
|
||||
QUEUE.get_or_init(RenameTailCleanupQueue::new)
|
||||
}
|
||||
|
||||
const EVENT_SET_DISK_RENAME_ROLLBACK: &str = "set_disk_rename_rollback";
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -3718,10 +4203,9 @@ async fn inspect_incomplete_rename_rollback(
|
||||
request.recreate_missing = Some(false);
|
||||
request.update_parity = Some(false);
|
||||
request.recursive = Some(false);
|
||||
match tokio::time::timeout(Duration::from_secs(1), submitter(request)).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()),
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(1), submitter(request))
|
||||
.await
|
||||
.unwrap_or_else(|_| ReadRepairAdmissionOutcome::Failed("rollback inspection admission timed out".to_string()))
|
||||
}
|
||||
|
||||
fn rename_rollback_task_outcome(
|
||||
@@ -4008,21 +4492,24 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
SubmitFuture,
|
||||
>(
|
||||
tail_drain: tokio::task::JoinHandle<Option<RenameTailOutcome>>,
|
||||
guard_release: tokio::sync::oneshot::Receiver<bool>,
|
||||
guard_release: oneshot::Receiver<bool>,
|
||||
guards: Guards,
|
||||
request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
|
||||
finalize: Finalize,
|
||||
cleanup: Cleanup,
|
||||
submit: Submit,
|
||||
) where
|
||||
Guards: Send,
|
||||
Guards: Send + 'static,
|
||||
Finalize: FnOnce() -> FinalizeFuture + Send,
|
||||
FinalizeFuture: Future<Output = ()> + Send,
|
||||
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>) -> CleanupFuture + Send,
|
||||
CleanupFuture: Future<Output = ()> + Send,
|
||||
Cleanup: 'static,
|
||||
CleanupFuture: 'static,
|
||||
Submit: FnOnce(rustfs_heal_contracts::heal_channel::HealChannelRequest) -> SubmitFuture + Send,
|
||||
SubmitFuture: Future<Output = ()> + Send,
|
||||
{
|
||||
let tail_async_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await {
|
||||
Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true),
|
||||
Ok(None) => {
|
||||
@@ -4051,16 +4538,80 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
(true, Vec::new(), false)
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
|
||||
tail_async_started,
|
||||
);
|
||||
// A dropped sender means the request continuation was cancelled. Cleanup
|
||||
// must still run; only the explicit `false` used by the hard-crash harness
|
||||
// suppresses all in-process continuation work.
|
||||
let guard_release_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
if matches!(guard_release.await, Ok(false)) {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
|
||||
guard_release_wait_started,
|
||||
);
|
||||
drop(guards);
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
|
||||
guard_release_wait_started,
|
||||
);
|
||||
let finalize_started = rustfs_io_metrics::put_stage_timer();
|
||||
finalize().await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE,
|
||||
finalize_started,
|
||||
);
|
||||
if tail_complete {
|
||||
cleanup(guards, tail_cleanup).await;
|
||||
let cleanup_submit_started = rustfs_io_metrics::put_stage_timer();
|
||||
if rename_tail_cleanup_counterfactual_skip_enabled() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP,
|
||||
1.0,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
|
||||
cleanup_submit_started,
|
||||
);
|
||||
drop(guards);
|
||||
} else if rename_tail_cleanup_defer_enabled() {
|
||||
let targets = tail_cleanup.len();
|
||||
let job = RenameTailCleanupJob {
|
||||
enqueued_at: Instant::now(),
|
||||
targets,
|
||||
future: Box::pin(cleanup(guards, tail_cleanup)),
|
||||
};
|
||||
if let Err(job) = rename_tail_cleanup_queue().try_enqueue(job).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
|
||||
cleanup_submit_started,
|
||||
);
|
||||
let cleanup_started = rustfs_io_metrics::put_stage_timer();
|
||||
job.future.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
|
||||
cleanup_started,
|
||||
);
|
||||
} else {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
|
||||
cleanup_submit_started,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
|
||||
cleanup_submit_started,
|
||||
);
|
||||
let cleanup_started = rustfs_io_metrics::put_stage_timer();
|
||||
cleanup(guards, tail_cleanup).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
|
||||
cleanup_started,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// The outer coordinator may fail while a disk rename still owns the
|
||||
// staging source. Preserve it for later healing/GC instead of racing
|
||||
@@ -4068,7 +4619,12 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
drop(guards);
|
||||
}
|
||||
if needs_heal {
|
||||
let heal_submit_started = rustfs_io_metrics::put_stage_timer();
|
||||
submit(request).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT,
|
||||
heal_submit_started,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4122,8 +4678,8 @@ impl SetDisks {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_quorum: usize,
|
||||
) -> crate::error::Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
|
||||
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<Option<SnapshotLeaseToken>>)> {
|
||||
let fence_path = disk::quota_mutation_fence_path(bucket, object);
|
||||
let results = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let fence_path = fence_path.clone();
|
||||
@@ -4165,8 +4721,8 @@ impl SetDisks {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_quorum: usize,
|
||||
) -> crate::error::Result<()> {
|
||||
let fence_path = crate::disk::quota_mutation_fence_path(bucket, object);
|
||||
) -> Result<()> {
|
||||
let fence_path = disk::quota_mutation_fence_path(bucket, object);
|
||||
let results = join_all(disks.iter().zip(tokens).filter_map(|(disk, token)| {
|
||||
let disk = disk.as_ref()?.clone();
|
||||
let token = (*token)?;
|
||||
@@ -4195,11 +4751,25 @@ impl SetDisks {
|
||||
errs: &[Option<DiskError>],
|
||||
cleanup_data_dirs: &[Option<Uuid>],
|
||||
old_current_sizes: &[Option<OldCurrentSize>],
|
||||
old_current_source_checked: &[bool],
|
||||
old_current_sources: &[Option<FileInfo>],
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<RenameDataCommit> {
|
||||
let data_dir = Self::reduce_common_data_dir(cleanup_data_dirs, write_quorum);
|
||||
let convergence = Self::classify_rename_convergence(disk_versions, errs);
|
||||
let old_current_size = Self::reduce_common_old_current_size(old_current_sizes, write_quorum);
|
||||
let old_current_source_quorum_checked =
|
||||
old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum;
|
||||
let old_current_source_all_successful_checked = errs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, err)| err.is_none())
|
||||
.all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false));
|
||||
let old_current_source = if old_current_source_quorum_checked {
|
||||
Self::reduce_common_old_current_source(old_current_sources, write_quorum)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let online_disks = Self::eval_disks(disks, errs);
|
||||
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
|
||||
let committed_file_info = file_infos.get(committed_slot).cloned().ok_or(DiskError::Unexpected)?;
|
||||
@@ -4227,6 +4797,9 @@ impl SetDisks {
|
||||
data_dir,
|
||||
cleanup_disks,
|
||||
old_current_size,
|
||||
old_current_source_checked: old_current_source_quorum_checked,
|
||||
old_current_source_all_successful_checked,
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
})
|
||||
@@ -4344,9 +4917,10 @@ impl SetDisks {
|
||||
let src_object = Arc::new(src_object.to_string());
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
let (commit_tx, commit_rx) = tokio::sync::oneshot::channel();
|
||||
let (commit_tx, commit_rx) = oneshot::channel();
|
||||
|
||||
let coordinator_failure_receipt = rollback_receipt.clone();
|
||||
let early_ack_spawn_to_quorum_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let tail_drain = tokio::spawn({
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
@@ -4379,6 +4953,11 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE
|
||||
};
|
||||
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
// Clone FileInfo and set erasure.index for this disk
|
||||
@@ -4398,6 +4977,7 @@ impl SetDisks {
|
||||
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4405,7 +4985,7 @@ impl SetDisks {
|
||||
&file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
crate::disk::RenameDataGuards {
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
@@ -4426,6 +5006,7 @@ impl SetDisks {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
|
||||
duration_ms,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms);
|
||||
let position = if result.is_ok() {
|
||||
let rank = successful_rename_completion_rank
|
||||
.as_ref()
|
||||
@@ -4464,9 +5045,12 @@ impl SetDisks {
|
||||
let mut data_dirs = vec![None; disk_count];
|
||||
let mut cleanup_data_dirs = vec![None; disk_count];
|
||||
let mut old_current_sizes = vec![None; disk_count];
|
||||
let mut old_current_source_checked = vec![false; disk_count];
|
||||
let mut old_current_sources = vec![None; disk_count];
|
||||
let mut capacity_scope_generation = None;
|
||||
let mut cleanup_at_snapshot = vec![false; disk_count];
|
||||
|
||||
let mut tail_after_quorum_started = None;
|
||||
while let Some(joined) = tasks.join_next().await {
|
||||
results_seen += 1;
|
||||
match joined {
|
||||
@@ -4476,6 +5060,8 @@ impl SetDisks {
|
||||
cleanup_data_dirs[idx] = res.cleanup_data_dir;
|
||||
disk_versions[idx] = res.sign;
|
||||
old_current_sizes[idx] = res.old_current_size;
|
||||
old_current_source_checked[idx] = res.old_current_source_checked;
|
||||
old_current_sources[idx] = res.old_current_source;
|
||||
errs[idx] = None;
|
||||
success_count += 1;
|
||||
}
|
||||
@@ -4501,6 +5087,8 @@ impl SetDisks {
|
||||
&errs,
|
||||
&cleanup_data_dirs,
|
||||
&old_current_sizes,
|
||||
&old_current_source_checked,
|
||||
&old_current_sources,
|
||||
write_quorum,
|
||||
);
|
||||
if let Ok(commit) = snapshot_commit.as_ref() {
|
||||
@@ -4511,8 +5099,13 @@ impl SetDisks {
|
||||
let _ = commit_tx.send(snapshot_commit);
|
||||
}
|
||||
sent_commit = true;
|
||||
tail_after_quorum_started = rustfs_io_metrics::put_stage_timer();
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT,
|
||||
tail_after_quorum_started,
|
||||
);
|
||||
|
||||
#[cfg(test)]
|
||||
rollback_fault_injection::after_fanout(&fanout_dst_object);
|
||||
@@ -4632,6 +5225,10 @@ impl SetDisks {
|
||||
}
|
||||
});
|
||||
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
|
||||
early_ack_spawn_to_quorum_wait_started,
|
||||
);
|
||||
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let commit = match commit_rx.await {
|
||||
Ok(commit) => commit,
|
||||
@@ -4644,8 +5241,13 @@ impl SetDisks {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
|
||||
quorum_wait_started,
|
||||
);
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
);
|
||||
commit
|
||||
})
|
||||
}
|
||||
@@ -4768,6 +5370,11 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE
|
||||
};
|
||||
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
let mut local_file_info;
|
||||
@@ -4801,6 +5408,7 @@ impl SetDisks {
|
||||
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4808,7 +5416,7 @@ impl SetDisks {
|
||||
file_info,
|
||||
&dst_bucket,
|
||||
&dst_object,
|
||||
crate::disk::RenameDataGuards {
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
@@ -4829,6 +5437,7 @@ impl SetDisks {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
|
||||
duration_ms,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(disk_wait_location_stage, duration_ms);
|
||||
let position = if result.is_ok() {
|
||||
let rank = successful_rename_completion_rank
|
||||
.as_ref()
|
||||
@@ -4864,9 +5473,16 @@ impl SetDisks {
|
||||
let mut data_dirs = vec![None; disk_count];
|
||||
let mut cleanup_data_dirs = vec![None; disk_count];
|
||||
let mut old_current_sizes = vec![None; disk_count];
|
||||
let mut old_current_source_checked = vec![false; disk_count];
|
||||
let mut old_current_sources = vec![None; disk_count];
|
||||
|
||||
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let sync_full_fanout_started = rustfs_io_metrics::put_stage_timer();
|
||||
let fanout_result = fanout.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT,
|
||||
sync_full_fanout_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
|
||||
quorum_wait_started,
|
||||
@@ -4907,6 +5523,8 @@ impl SetDisks {
|
||||
cleanup_data_dirs[idx] = res.cleanup_data_dir;
|
||||
disk_versions[idx].clone_from(&res.sign);
|
||||
old_current_sizes[idx] = res.old_current_size;
|
||||
old_current_source_checked[idx] = res.old_current_source_checked;
|
||||
old_current_sources[idx] = res.old_current_source.clone();
|
||||
errs.push(None);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
@@ -5043,6 +5661,18 @@ impl SetDisks {
|
||||
let data_dir = Self::reduce_common_data_dir(&cleanup_data_dirs, write_quorum);
|
||||
let convergence = Self::classify_rename_convergence(&disk_versions, &errs);
|
||||
let old_current_size = Self::reduce_common_old_current_size(&old_current_sizes, write_quorum);
|
||||
let old_current_source_quorum_checked =
|
||||
old_current_source_checked.iter().filter(|checked| **checked).count() >= write_quorum;
|
||||
let old_current_source_all_successful_checked = errs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, err)| err.is_none())
|
||||
.all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false));
|
||||
let old_current_source = if old_current_source_quorum_checked {
|
||||
Self::reduce_common_old_current_source(&old_current_sources, write_quorum)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let online_disks = Self::eval_disks(disks, &errs);
|
||||
let committed_slot = online_disks.iter().position(Option::is_some).ok_or(DiskError::Unexpected)?;
|
||||
let committed_file_info = std::mem::take(&mut file_infos[committed_slot]);
|
||||
@@ -5070,11 +5700,34 @@ impl SetDisks {
|
||||
data_dir,
|
||||
cleanup_disks,
|
||||
old_current_size,
|
||||
old_current_source_checked: old_current_source_quorum_checked,
|
||||
old_current_source_all_successful_checked,
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn reduce_common_old_current_source(
|
||||
old_current_sources: &[Option<FileInfo>],
|
||||
write_quorum: usize,
|
||||
) -> Option<FileInfo> {
|
||||
let mut counts: HashMap<[u8; 32], (usize, FileInfo)> = HashMap::new();
|
||||
|
||||
for source in old_current_sources.iter().flatten() {
|
||||
let entry = counts
|
||||
.entry(Self::file_info_quorum_hash(source))
|
||||
.or_insert_with(|| (0, source.clone()));
|
||||
entry.0 += 1;
|
||||
}
|
||||
|
||||
counts
|
||||
.into_values()
|
||||
.filter(|(count, _)| *count >= write_quorum)
|
||||
.max_by_key(|(count, _)| *count)
|
||||
.map(|(_, source)| source)
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1009: reduce the per-disk observations of the
|
||||
/// destination's previous current version to one set-level value, mirroring
|
||||
/// `reduce_common_data_dir`: the observation reported by at least
|
||||
@@ -7546,9 +8199,9 @@ mod tests {
|
||||
);
|
||||
scope.try_begin().expect("delete scope should enter flight");
|
||||
let scope_guard = crate::object_api::ScannerPublicationCommitScopeGuard::new(scope.clone());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let (finished_tx, finished_rx) = oneshot::channel();
|
||||
|
||||
let waiter = tokio::spawn(run_scanner_publication_delete_owner(Some(scope.clone()), move || async move {
|
||||
started_tx.send(()).expect("delete owner should start");
|
||||
@@ -9978,7 +10631,7 @@ mod tests {
|
||||
#[allow(unreachable_code)]
|
||||
None
|
||||
});
|
||||
let (release, released) = tokio::sync::oneshot::channel::<bool>();
|
||||
let (release, released) = oneshot::channel::<bool>();
|
||||
drop(release);
|
||||
|
||||
finish_rename_tail_heal(
|
||||
@@ -11185,7 +11838,7 @@ mod tests {
|
||||
info.size = 11;
|
||||
info.parts.clear();
|
||||
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("physical publication fixture requires local disks");
|
||||
};
|
||||
// Linux IO paths use a mount FD, which is also the namespace lock key.
|
||||
@@ -11193,7 +11846,7 @@ mod tests {
|
||||
.get_disk()
|
||||
.get_object_path_for_io(bucket, object)
|
||||
.expect("the publication path must resolve through the disk's mount lease");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (entered_tx, entered_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
hooks.push(os::prepared_publication_test_hooks::install(
|
||||
&destination.join(STORAGE_FORMAT_FILE),
|
||||
@@ -11379,11 +12032,11 @@ mod tests {
|
||||
info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None);
|
||||
}
|
||||
let disk = disks[3].as_ref().expect("tail disk");
|
||||
let crate::disk::Disk::Local(local) = disk.as_ref() else {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("local fixture");
|
||||
};
|
||||
let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (entered_tx, entered_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || {
|
||||
let _ = entered_tx.send(());
|
||||
@@ -11395,7 +12048,7 @@ mod tests {
|
||||
let receipt = RenameRollbackReceipt::default();
|
||||
let caller_receipt = receipt.clone();
|
||||
let caller_disks = disks.clone();
|
||||
let (ack_tx, ack_rx) = tokio::sync::oneshot::channel();
|
||||
let (ack_tx, ack_rx) = oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let commit = SetDisks::rename_data_owned_with_fence(
|
||||
&caller_disks,
|
||||
|
||||
@@ -166,7 +166,7 @@ use sha2::Sha256;
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::mem::{self};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
@@ -410,6 +410,12 @@ impl Drop for ObjectLockDiagGuard {
|
||||
}
|
||||
|
||||
let hold = self.acquired_at.elapsed();
|
||||
if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
|
||||
hold.as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
record_object_lock_diag_hold_duration(self.op, self.mode, hold);
|
||||
let threshold = get_object_lock_diag_slow_hold_threshold();
|
||||
if hold >= threshold {
|
||||
@@ -3901,6 +3907,9 @@ pub struct SetDisks {
|
||||
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
|
||||
/// a set share one generation marker.
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
/// Per-set proof that successful rename_data peers return the old-current
|
||||
/// source capability needed before PUT may skip its pre-rename lookup.
|
||||
tier_free_version_rename_data_source_capable: Arc<AtomicBool>,
|
||||
/// Orphan prefixes whose last purge scan met data that can never be
|
||||
/// purged by listing (residue without a committed marker, an in-flight
|
||||
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
|
||||
@@ -4697,6 +4706,7 @@ impl SetDisks {
|
||||
ctx,
|
||||
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
|
||||
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
|
||||
tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)),
|
||||
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
|
||||
@@ -4711,6 +4721,15 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool {
|
||||
self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) {
|
||||
self.tier_free_version_rename_data_source_capable
|
||||
.store(capable, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
self.local_lock_manager = ctx.lock_manager();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,11 @@ use crate::disk::error::{DiskError, Result as DiskResult};
|
||||
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use rustfs_io_metrics::{
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
|
||||
};
|
||||
use tracing::{debug, error};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
@@ -293,15 +298,38 @@ async fn rename_local_data_with_ctx(
|
||||
destination: (&str, &str),
|
||||
mut guards: RenameDataGuards,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
let lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = lookup_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms);
|
||||
}
|
||||
let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
|
||||
let admit_started = rustfs_io_metrics::put_stage_timer();
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
|
||||
let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = admit_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms);
|
||||
}
|
||||
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let local_rename_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||
.await
|
||||
.result;
|
||||
let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = local_rename_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms);
|
||||
}
|
||||
drop(owner);
|
||||
if let Some(handler_started) = handler_started {
|
||||
let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms);
|
||||
let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0);
|
||||
let overhead_ms = (handler_ms - child_ms).max(0.0);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
|
||||
@@ -20,8 +20,8 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, enable_safe_no_quota_cache_fast_path_for_test,
|
||||
init_bucket_metadata_sys, isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
@@ -170,6 +170,103 @@ async fn reused_metadata_lock_still_enforces_quota() {
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn safe_no_quota_cache_fast_path_does_not_ignore_later_quota() {
|
||||
let _guard = enable_safe_no_quota_cache_fast_path_for_test();
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "safe-no-quota-cache-quota-set";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let mut first = PutObjReader::from_vec(b"cache no quota".to_vec());
|
||||
store
|
||||
.put_object(bucket, "warm", &mut first, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("warm no-quota metadata cache");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota after no-quota cache was observed");
|
||||
|
||||
let mut opts = quota_snapshot_options(&store, bucket).await;
|
||||
assert!(opts.set_quota_admission(0, 1));
|
||||
let mut data = PutObjReader::from_vec(b"too large".to_vec());
|
||||
let error = store
|
||||
.put_object(bucket, "object", &mut data, &opts)
|
||||
.await
|
||||
.expect_err("quota set after a no-quota cache hit must still reject growth");
|
||||
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn safe_no_quota_cache_fast_path_allows_write_after_quota_is_cleared() {
|
||||
let _guard = enable_safe_no_quota_cache_fast_path_for_test();
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "safe-no-quota-cache-quota-cleared";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", Vec::new())
|
||||
.await
|
||||
.expect("clear quota");
|
||||
|
||||
let mut data = PutObjReader::from_vec(b"allowed after clear".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("cleared quota should allow writes through the no-quota fast path");
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, "object", None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read object written after quota clear");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read body");
|
||||
assert_eq!(bytes, b"allowed after clear");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn safe_no_quota_cache_fast_path_fails_closed_on_invalid_quota_json() {
|
||||
let _guard = enable_safe_no_quota_cache_fast_path_for_test();
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "safe-no-quota-cache-invalid-quota";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", b"{not-json".to_vec())
|
||||
.await
|
||||
.expect("persist invalid quota bytes for fail-closed read");
|
||||
|
||||
let mut data = PutObjReader::from_vec(b"must fail".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("invalid quota JSON must not degrade to no-quota");
|
||||
}
|
||||
|
||||
async fn quota_snapshot_options(store: &Arc<storage_api::ECStore>, bucket: &str) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_put_releases_shared_metadata_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -15,7 +15,9 @@ pub(crate) mod metadata_lock {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{
|
||||
PutObjectCommitBarrier, PutObjectCommitPause, enable_safe_no_quota_cache_fast_path_for_test,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod contract_compat {
|
||||
|
||||
@@ -110,8 +110,195 @@ pub fn put_stage_timer() -> Option<std::time::Instant> {
|
||||
}
|
||||
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT: &str = "put_object_commit_namespace_lock_wait";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD: &str = "put_object_commit_namespace_lock_held";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME: &str = "put_object_commit_namespace_lock_held_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE: &str =
|
||||
"put_object_commit_namespace_lock_held_pre_rename_prepare";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF: &str =
|
||||
"put_object_commit_namespace_lock_held_rename_result_to_tail_handoff";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF: &str =
|
||||
"put_object_commit_namespace_lock_held_tail_handoff";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME: &str = "put_object_commit_namespace_lock_held_post_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE: &str =
|
||||
"put_object_commit_namespace_lock_held_guard_release";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_fence_release";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_commit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT: &str = "put_object_commit_namespace_lock_held_heal_submit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT: &str =
|
||||
"put_object_commit_namespace_lock_held_cleanup_receipt";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE: &str =
|
||||
"put_object_commit_namespace_lock_held_metadata_invalidate";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD: &str =
|
||||
"put_object_commit_namespace_lock_held_object_info_build";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN: &str =
|
||||
"put_object_commit_namespace_lock_held_post_object_info_to_return";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT: &str = "put_object_commit_task_wait";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL: &str = "put_object_commit_future_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT: &str = "put_object_commit_operation_select_wait";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL: &str = "put_object_operation_await_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT: &str = "put_object_operation_body_to_commit_submit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN: &str = "put_object_operation_result_return";
|
||||
pub const PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY: &str = "put_object_operation_first_poll_delay";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY: &str =
|
||||
"put_object_commit_namespace_lock_acquired_to_commit_ready";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP: &str =
|
||||
"put_object_commit_ready_precondition_and_timestamp";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP: &str = "put_object_commit_ready_expected_version_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP: &str = "put_object_commit_ready_explicit_version_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY: &str = "put_object_commit_ready_restore_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY: &str = "put_object_commit_ready_fence_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE: &str =
|
||||
"put_object_commit_ready_decommission_capacity_fence";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ: &str =
|
||||
"put_object_commit_ready_transaction_epoch_fence_read";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN: &str = "put_object_commit_ready_quota_begin";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST: &str = "put_object_commit_ready_metadata_adjust";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE: &str = "put_object_commit_ready_quota_reserve";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE: &str =
|
||||
"put_object_commit_ready_quota_mutation_fence_prepare";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE: &str = "put_object_commit_ready_context_prepare";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH: &str = "put_object_quota_begin_meta_bucket_fast_path";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK: &str = "put_object_quota_begin_metadata_lock";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP: &str = "put_object_quota_begin_cache_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ: &str = "put_object_quota_begin_config_read";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ_COUNTERFACTUAL_SKIP: &str =
|
||||
"put_object_quota_begin_config_read_counterfactual_skip";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_HIT: &str = "put_object_quota_begin_safe_no_quota_cache_hit";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK: &str = "put_object_quota_begin_lock_lost_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE: &str = "put_object_quota_begin_policy_evaluate";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF: &str = "put_object_quota_begin_capability_proof";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION: &str = "put_object_quota_begin_snapshot_admission";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP: &str = "put_object_quota_begin_object_store_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD: &str = "put_object_quota_begin_context_build";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER: &str = "put_object_commit_submit_to_closure_enter";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT: &str = "put_object_commit_closure_enter_to_first_await";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL: &str = "put_object_commit_closure_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME: &str = "put_object_commit_before_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL: &str = "put_object_commit_pre_rename_await_total";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED: &str =
|
||||
"put_object_commit_pre_rename_quota_mark_commit_started";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK: &str = "put_object_commit_pre_rename_first_fence_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY: &str = "put_object_commit_pre_rename_restore_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY: &str =
|
||||
"put_object_commit_pre_rename_transaction_fence_verify";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK: &str = "put_object_commit_pre_rename_second_fence_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK: &str = "put_object_commit_pre_rename_scanner_scope_check";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP: &str =
|
||||
"put_object_commit_pre_rename_tier_free_version_source_lookup";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES: &str =
|
||||
"put_object_commit_pre_rename_assign_rename_indexes";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME: &str = "put_object_commit_pre_rename_post_await_to_rename";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN: &str = "put_object_commit_after_rename_to_return";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT: &str = "set_disk_rename_sync_full_fanout_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT: &str = "set_disk_rename_early_ack_spawn_to_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER: &str = "set_disk_rename_early_ack_attach_tail_owner";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT: &str = "set_disk_rename_tail_after_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT: &str = "set_disk_rename_tail_async_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT: &str = "set_disk_rename_tail_guard_release_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE: &str = "set_disk_rename_tail_finalize";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP: &str = "set_disk_rename_tail_cleanup";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT: &str = "set_disk_rename_tail_cleanup_submit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP: &str = "set_disk_rename_tail_cleanup_counterfactual_skip";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED: &str = "set_disk_rename_tail_cleanup_defer_enqueued";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC: &str = "set_disk_rename_tail_cleanup_defer_fallback_sync";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_defer_queue_push";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_defer_queue_depth";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_queue_depth_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_defer_oldest_age";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_oldest_age_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS: &str = "set_disk_rename_tail_cleanup_defer_targets";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY: &str = "set_disk_rename_tail_cleanup_defer_start_delay";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG: &str = "set_disk_rename_tail_cleanup_defer_completed_lag";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER: &str = "set_disk_rename_tail_cleanup_defer_hold_worker";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY: &str = "set_disk_rename_tail_cleanup_defer_worker_delay";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT: &str = "set_disk_rename_tail_cleanup_per_disk_permit_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT: &str = "set_disk_rename_tail_cleanup_per_disk_inflight";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD: &str = "set_disk_rename_tail_cleanup_foreground_yield";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE: &str =
|
||||
"set_disk_rename_tail_cleanup_deferred_foreground_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE: &str = "set_disk_rename_foreground_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW: &str = "set_disk_rename_tail_cleanup_defer_batch_window";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_defer_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_burst_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_defer_worker_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_worker_poll_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED: &str = "set_disk_rename_tail_cleanup_defer_completed";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION: &str =
|
||||
"set_disk_rename_tail_cleanup_defer_worker_duration";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH: &str = "set_disk_rename_tail_cleanup_lane_queue_depth";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH: &str = "set_disk_rename_tail_cleanup_lane_queue_push";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_queue_depth_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE: &str = "set_disk_rename_tail_cleanup_lane_oldest_age";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_oldest_age_max_sample";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE: &str = "set_disk_rename_tail_cleanup_lane_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_burst_batch_size";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE: &str = "set_disk_rename_tail_cleanup_lane_worker_active";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_worker_poll_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME: &str = "set_disk_rename_tail_cleanup_lane_service_time";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE: &str = "set_disk_rename_tail_cleanup_lane_requeue";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT: &str =
|
||||
"set_disk_rename_tail_cleanup_lane_foreground_active_hit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT: &str = "set_disk_rename_tail_heal_submit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL: &str = "set_disk_rename_disk_wait_local";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE: &str = "set_disk_rename_disk_wait_remote";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE: &str = "set_disk_rename_remote_client_prepare";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC: &str = "set_disk_rename_remote_client_rpc";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT: &str = "set_disk_rename_remote_client_rpc_await";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT: &str =
|
||||
"set_disk_rename_remote_client_rpc_await_poll_pending_count";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY: &str =
|
||||
"set_disk_rename_remote_client_rpc_await_first_pending_to_ready";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE: &str = "set_disk_rename_remote_client_request_scope";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL: &str = "set_disk_rename_remote_client_transport_call";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT: &str =
|
||||
"set_disk_rename_remote_client_transport_call_poll_pending_count";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY: &str =
|
||||
"set_disk_rename_remote_client_transport_call_first_pending_to_ready";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE: &str = "set_disk_rename_remote_client_replay_response";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER: &str = "set_disk_rename_remote_client_into_inner";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK: &str = "set_disk_rename_remote_client_success_check";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE: &str = "set_disk_rename_remote_client_decode";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS: &str = "set_disk_rename_remote_client_response_process";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS: &str = "set_disk_rename_remote_client_response_headers";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE: &str = "set_disk_rename_remote_service_response_encode";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_fence_release_triggered";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED: &str =
|
||||
"put_object_commit_namespace_lock_held_quota_fence_release_skipped";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED: &str =
|
||||
"put_object_commit_namespace_lock_held_heal_submit_triggered";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED: &str =
|
||||
"put_object_commit_namespace_lock_held_heal_submit_skipped";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED: &str =
|
||||
"put_object_commit_namespace_lock_held_cleanup_receipt_triggered";
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED: &str =
|
||||
"put_object_commit_namespace_lock_held_cleanup_receipt_skipped";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL: &str = "set_disk_rename_remote_handler_total";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD: &str = "set_disk_rename_remote_handler_overhead";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP: &str = "set_disk_rename_remote_handler_lookup";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT: &str = "set_disk_rename_remote_handler_admit";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME: &str = "set_disk_rename_remote_handler_local_rename";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY: &str = "set_disk_rename_remote_service_entry";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER: &str = "set_disk_rename_remote_service_before_handler";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT: &str = "set_disk_rename_remote_service_handler_await";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER: &str = "set_disk_rename_remote_service_after_handler";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD: &str = "set_disk_rename_remote_service_response_build";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN: &str = "set_disk_rename_remote_service_response_return";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE: &str = "set_disk_rename_remote_transport_service_future";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME: &str = "set_disk_rename_remote_transport_first_frame";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE: &str = "set_disk_rename_remote_transport_body_complete";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync";
|
||||
@@ -2072,6 +2259,14 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_quota_cache_decision(decision: &'static str, reason: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_s3_put_object_quota_cache_total", "decision" => decision, "reason" => reason).increment(1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
@@ -3171,8 +3366,154 @@ mod tests {
|
||||
assert_eq!(PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, "put_object_commit_namespace_lock_wait");
|
||||
let stages = [
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_PRE_RENAME_PREPARE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_RENAME_RESULT_TO_TAIL_HANDOFF,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_TAIL_HANDOFF,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_GUARD_RELEASE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_COMMIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_METADATA_INVALIDATE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_OBJECT_INFO_BUILD,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_POST_OBJECT_INFO_TO_RETURN,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN,
|
||||
PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_PRECONDITION_AND_TIMESTAMP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPECTED_VERSION_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_EXPLICIT_VERSION_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_RESTORE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_FENCE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_DECOMMISSION_CAPACITY_FENCE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_TRANSACTION_EPOCH_FENCE_READ,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_BEGIN,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_METADATA_ADJUST,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_RESERVE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_QUOTA_MUTATION_FENCE_PREPARE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_READY_CONTEXT_PREPARE,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ_COUNTERFACTUAL_SKIP,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_HIT,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_AWAIT_TOTAL,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_QUOTA_MARK_COMMIT_STARTED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_FIRST_FENCE_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_RESTORE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TRANSACTION_FENCE_VERIFY,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SECOND_FENCE_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_SCANNER_SCOPE_CHECK,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_TIER_FREE_VERSION_SOURCE_LOOKUP,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_ASSIGN_RENAME_INDEXES,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_PRE_RENAME_POST_AWAIT_TO_RENAME,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN,
|
||||
PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_SYNC_FULL_FANOUT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_AFTER_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_GUARD_RELEASE_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_FINALIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_SUBMIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_ENQUEUED,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_FALLBACK_SYNC,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_PUSH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_OLDEST_AGE_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_TARGETS,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_START_DELAY,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED_LAG,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DELAY,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_PERMIT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_PER_DISK_INFLIGHT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_FOREGROUND_YIELD,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFERRED_FOREGROUND_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_FOREGROUND_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_WINDOW,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_BURST_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_POLL_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_COMPLETED,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_DEFER_WORKER_DURATION,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_PUSH,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_QUEUE_DEPTH_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_OLDEST_AGE_MAX_SAMPLE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_BURST_BATCH_SIZE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_ACTIVE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_WORKER_POLL_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_SERVICE_TIME,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_REQUEUE,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_LANE_FOREGROUND_ACTIVE_HIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_TAIL_HEAL_SUBMIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_REMOTE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_TRIGGERED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_QUOTA_FENCE_RELEASE_SKIPPED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_TRIGGERED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_HEAL_SUBMIT_SKIPPED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_TRIGGERED,
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD_CLEANUP_RECEIPT_SKIPPED,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE,
|
||||
PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC,
|
||||
@@ -3185,7 +3526,26 @@ mod tests {
|
||||
let unique = stages.iter().copied().collect::<HashSet<_>>();
|
||||
assert_eq!(unique.len(), stages.len());
|
||||
assert!(stages.iter().all(|stage| {
|
||||
(stage.starts_with("set_disk_rename_") || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT)
|
||||
(stage.starts_with("set_disk_rename_")
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_TASK_WAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_FUTURE_TOTAL
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_OPERATION_SELECT_WAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_AWAIT_TOTAL
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_BODY_TO_COMMIT_SUBMIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_RESULT_RETURN
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_OPERATION_FIRST_POLL_DELAY
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_ACQUIRED_TO_COMMIT_READY
|
||||
|| stage.starts_with("put_object_commit_ready_")
|
||||
|| stage.starts_with("put_object_quota_begin_")
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_SUBMIT_TO_CLOSURE_ENTER
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_ENTER_TO_FIRST_AWAIT
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_CLOSURE_TOTAL
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_BEFORE_RENAME
|
||||
|| stage.starts_with("put_object_commit_pre_rename_")
|
||||
|| *stage == PUT_STAGE_PUT_OBJECT_COMMIT_AFTER_RENAME_TO_RETURN
|
||||
|| stage.starts_with("put_object_commit_namespace_lock_held_"))
|
||||
&& !stage.contains('/')
|
||||
&& !stage.contains('{')
|
||||
}));
|
||||
|
||||
@@ -221,6 +221,16 @@ const GET_OBJECT_STAGE_LIFECYCLE_EXPIRATION: &str = "lifecycle_expiration";
|
||||
|
||||
const GET_OBJECT_STAGE_METADATA_FILTER: &str = "metadata_filter";
|
||||
|
||||
const GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT: &str = "body_cache_hook_hit";
|
||||
const GET_OBJECT_STAGE_BODY_CACHE_LOOKUP: &str = "body_cache_lookup";
|
||||
const GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF: &str = "body_cache_fill_handoff";
|
||||
const GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF: &str = "body_buffered_handoff";
|
||||
const GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD: &str = "output_struct_build";
|
||||
const GET_OBJECT_STAGE_PREPARE_READ_EXECUTION: &str = "prepare_read_execution";
|
||||
const GET_OBJECT_STAGE_RESPONSE_CORS_WRAP: &str = "response_cors_wrap";
|
||||
const GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT: &str = "response_header_inject";
|
||||
const GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE: &str = "response_event_complete";
|
||||
|
||||
const GET_OBJECT_STREAM_WARN_THRESHOLD: Duration = Duration::from_secs(5);
|
||||
|
||||
static GET_OBJECT_BUFFER_THRESHOLD_WARNED: AtomicBool = AtomicBool::new(false);
|
||||
@@ -499,6 +509,14 @@ fn get_object_stream_failure_reason(error_class: &'static str) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// A mid-stream reopen can only recover when another representation of the
|
||||
/// committed object may exist. Single-disk, no-parity objects have no alternate
|
||||
/// shard to discover after a bitrot/read-quorum failure, so arming the normal
|
||||
/// three-attempt resume loop only repeats the same failing read and adds delay.
|
||||
fn should_attach_get_object_resume(info: &ObjectInfo) -> bool {
|
||||
info.parity_blocks > 0 || info.data_blocks > 1 || !info.transitioned_object.tier.is_empty()
|
||||
}
|
||||
|
||||
fn record_get_object_reader_stream_failure(
|
||||
reason: &'static str,
|
||||
error_class: &'static str,
|
||||
@@ -3314,12 +3332,15 @@ impl DefaultObjectUsecase {
|
||||
// already supplied this body, the request-level plan was built before
|
||||
// the authoritative lookup. Serve it without planning a second time.
|
||||
if cache_hook_served && let Some(bytes) = buffered_body.take() {
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
let hook_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let body = Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
response_content_length,
|
||||
GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE,
|
||||
lifecycle,
|
||||
));
|
||||
);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, hook_hit_start);
|
||||
return Ok(body);
|
||||
}
|
||||
|
||||
if !cache_fill_allowed {
|
||||
@@ -3360,20 +3381,27 @@ impl DefaultObjectUsecase {
|
||||
// authoritative because it ran after fresh metadata resolution, so the
|
||||
// app layer skips its own lookup and only uses the plan to fill.
|
||||
if !cache_hook_probed {
|
||||
match lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await {
|
||||
let cache_lookup_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let cache_lookup = lookup_get_object_body_cache_hit(cache_adapter, &cache_plan).await;
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_LOOKUP, cache_lookup_start);
|
||||
match cache_lookup {
|
||||
GetObjectBodyCacheLookup::Hit(bytes) => {
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
let cache_hit_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let body = Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
response_content_length,
|
||||
GET_MEMORY_BODY_SOURCE_OBJECT_DATA_CACHE,
|
||||
lifecycle,
|
||||
));
|
||||
);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_HOOK_HIT, cache_hit_start);
|
||||
return Ok(body);
|
||||
}
|
||||
GetObjectBodyCacheLookup::Disabled | GetObjectBodyCacheLookup::Skip | GetObjectBodyCacheLookup::Miss => {}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(buffered_body) = buffered_body {
|
||||
let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
// ODC-15: the body is already fully in hand, so keep the fill off the
|
||||
// response's critical path. For a cacheable plan, run the fill in a
|
||||
// detached task (Bytes is a cheap clone) and return immediately. For
|
||||
@@ -3389,13 +3417,17 @@ impl DefaultObjectUsecase {
|
||||
} else if cache_fill_allowed {
|
||||
let _ = fill_get_object_body_cache_from_buffered_body(cache_adapter, &cache_plan, &buffered_body).await;
|
||||
}
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF, cache_fill_handoff_start);
|
||||
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
let buffered_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let body = Self::build_memory_bytes_blob(
|
||||
buffered_body,
|
||||
response_content_length,
|
||||
GET_MEMORY_BODY_SOURCE_BUFFERED_BODY,
|
||||
lifecycle,
|
||||
));
|
||||
);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_BUFFERED_HANDOFF, buffered_handoff_start);
|
||||
return Ok(body);
|
||||
}
|
||||
|
||||
let should_materialize_for_cache = cache_adapter.materialize_fill_enabled()
|
||||
@@ -3449,6 +3481,7 @@ impl DefaultObjectUsecase {
|
||||
{
|
||||
Ok(buf) => {
|
||||
let bytes = Bytes::from(buf);
|
||||
let cache_fill_handoff_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
// ODC-15: fill off the response's critical path (see the
|
||||
// buffered-body branch above).
|
||||
let cache_adapter = cache_adapter.clone();
|
||||
@@ -3457,6 +3490,10 @@ impl DefaultObjectUsecase {
|
||||
tokio::spawn(async move {
|
||||
let _ = fill_get_object_body_cache_from_materialized_body(&cache_adapter, &cache_plan, &fill_bytes).await;
|
||||
});
|
||||
record_get_object_s3_handler_stage_duration(
|
||||
GET_OBJECT_STAGE_BODY_CACHE_FILL_HANDOFF,
|
||||
cache_fill_handoff_start,
|
||||
);
|
||||
|
||||
return Ok(Self::build_memory_bytes_blob(
|
||||
bytes,
|
||||
@@ -3595,13 +3632,19 @@ impl DefaultObjectUsecase {
|
||||
None => helper,
|
||||
};
|
||||
let helper = helper.version_id(version_id_for_event);
|
||||
let cors_wrap_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let mut response = wrap_response_with_cors(bucket, method, headers, output).await;
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_CORS_WRAP, cors_wrap_start);
|
||||
let header_inject_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
inject_accept_ranges_header(&mut response.headers);
|
||||
// Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot
|
||||
// carry (#1257). This is the download-side integrity path AWS SDKs verify.
|
||||
inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_HEADER_INJECT, header_inject_start);
|
||||
let result = Ok(response);
|
||||
let event_complete_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let _ = helper.complete(&result);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_RESPONSE_EVENT_COMPLETE, event_complete_start);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -3712,6 +3755,7 @@ impl DefaultObjectUsecase {
|
||||
let metadata = filter_object_metadata(&info.user_defined);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_METADATA_FILTER, metadata_filter_start);
|
||||
|
||||
let output_struct_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let output = GetObjectOutput {
|
||||
body: Some(body),
|
||||
content_length: Some(response_content_length),
|
||||
@@ -3740,6 +3784,7 @@ impl DefaultObjectUsecase {
|
||||
storage_class,
|
||||
..Default::default()
|
||||
};
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_OUTPUT_STRUCT_BUILD, output_struct_start);
|
||||
|
||||
Ok(GetObjectOutputContext {
|
||||
output,
|
||||
@@ -4040,6 +4085,7 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let manager = get_concurrency_manager();
|
||||
|
||||
let prepare_read_start = stage_metrics_enabled.then(std::time::Instant::now);
|
||||
let mut prepared_read = self
|
||||
.prepare_get_object_read_execution(
|
||||
&req,
|
||||
@@ -4055,6 +4101,7 @@ impl DefaultObjectUsecase {
|
||||
object_traffic_health.clone(),
|
||||
)
|
||||
.await;
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_PREPARE_READ_EXECUTION, prepare_read_start);
|
||||
// An object missing locally (and only missing — other errors keep
|
||||
// their semantics) may still be served by a remote copy.
|
||||
if let Err(err) = &prepared_read
|
||||
@@ -4092,6 +4139,7 @@ impl DefaultObjectUsecase {
|
||||
return Self::complete_get_object_error(helper.version_id(version_id_for_event), err);
|
||||
}
|
||||
Some(OdmGetOutcome::RetryLocal) => {
|
||||
let prepare_read_retry_start = stage_metrics_enabled.then(std::time::Instant::now);
|
||||
prepared_read = self
|
||||
.prepare_get_object_read_execution(
|
||||
&req,
|
||||
@@ -4107,6 +4155,10 @@ impl DefaultObjectUsecase {
|
||||
object_traffic_health,
|
||||
)
|
||||
.await;
|
||||
record_get_object_s3_handler_stage_duration(
|
||||
GET_OBJECT_STAGE_PREPARE_READ_EXECUTION,
|
||||
prepare_read_retry_start,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4188,16 +4240,18 @@ impl DefaultObjectUsecase {
|
||||
part_number,
|
||||
lifecycle,
|
||||
|info| {
|
||||
Some(get_object_resume_control(GetObjectResumeContext::new(
|
||||
store,
|
||||
&bucket,
|
||||
&key,
|
||||
opts,
|
||||
&req.headers,
|
||||
info,
|
||||
resume_range_start,
|
||||
resume_range_end,
|
||||
)))
|
||||
should_attach_get_object_resume(info).then(|| {
|
||||
get_object_resume_control(GetObjectResumeContext::new(
|
||||
store,
|
||||
&bucket,
|
||||
&key,
|
||||
opts,
|
||||
&req.headers,
|
||||
info,
|
||||
resume_range_start,
|
||||
resume_range_end,
|
||||
))
|
||||
})
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -8365,6 +8419,34 @@ mod tests {
|
||||
assert_eq!(reopen_count.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_object_resume_only_attaches_for_recoverable_layouts() {
|
||||
let single_disk = ObjectInfo {
|
||||
data_blocks: 1,
|
||||
parity_blocks: 0,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!should_attach_get_object_resume(&single_disk));
|
||||
|
||||
let erasure = ObjectInfo {
|
||||
data_blocks: 2,
|
||||
parity_blocks: 1,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_attach_get_object_resume(&erasure));
|
||||
|
||||
let transitioned = ObjectInfo {
|
||||
data_blocks: 1,
|
||||
parity_blocks: 0,
|
||||
transitioned_object: crate::storage_api::server::event::contract::lifecycle::TransitionedObject {
|
||||
tier: "remote-tier".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_attach_get_object_resume(&transitioned));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_object_resume_identity_requires_same_version() {
|
||||
let version_id = Uuid::from_u128(0x1234);
|
||||
|
||||
@@ -1721,6 +1721,7 @@ impl DefaultObjectUsecase {
|
||||
)?;
|
||||
}
|
||||
|
||||
let checksum_input_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let mut md5hex = match content_md5 {
|
||||
Some(PutObjectContentMd5::Base64(base64_md5)) => {
|
||||
let md5 = base64_simd::STANDARD
|
||||
@@ -1733,12 +1734,15 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
let mut sha256hex = get_content_sha256_with_query(headers, query);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_checksum_input_prepare", checksum_input_stage_start);
|
||||
|
||||
let mut write_plan = WritePlan::new();
|
||||
// Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject
|
||||
// response (#1256); captured at want_checksum set points before opts is moved.
|
||||
let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new();
|
||||
let reader_prepare_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let mut reader = if should_compress {
|
||||
let compressed_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
@@ -1761,51 +1765,98 @@ impl DefaultObjectUsecase {
|
||||
|
||||
size = HashReader::SIZE_PRESERVE_LAYER;
|
||||
write_plan = write_plan.with_compression(algorithm);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_reader_prepare_compressed",
|
||||
compressed_reader_stage_start,
|
||||
);
|
||||
hrd
|
||||
} else {
|
||||
if use_zero_copy_eager_put_path {
|
||||
let zero_copy_start = std::time::Instant::now();
|
||||
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_body = read_zero_copy_put_body_exact(body, body_size as usize).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_body_eager_read_zero_copy", eager_read_stage_start);
|
||||
rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let reader =
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_hash_reader_setup_zero_copy",
|
||||
hash_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
} else if use_empty_or_small_eager_put_path {
|
||||
if (body_size as usize) <= POOL_BYPASS_MAX_SIZE {
|
||||
// Bypass BytesPool for very small objects to avoid Small-tier
|
||||
// Mutex contention under high concurrency. Direct allocation
|
||||
// for ≤4KiB is negligible cost.
|
||||
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_body = read_small_put_body_exact_direct(
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
body_size as usize,
|
||||
)
|
||||
.await?;
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_body_eager_read_small_direct",
|
||||
eager_read_stage_start,
|
||||
);
|
||||
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let reader = HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false)
|
||||
.map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_hash_reader_setup_small_direct",
|
||||
hash_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
} else {
|
||||
let pool = get_concurrency_manager().bytes_pool();
|
||||
let eager_read_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_body = read_small_put_body_exact_pooled(
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
body_size as usize,
|
||||
pool.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_body_eager_read_small_pooled",
|
||||
eager_read_stage_start,
|
||||
);
|
||||
let hash_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let eager_reader = PooledBufferReader::new(eager_body, body_size as usize);
|
||||
HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
let reader = HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false)
|
||||
.map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_hash_reader_setup_small_pooled",
|
||||
hash_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
}
|
||||
} else {
|
||||
let streaming_reader_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
);
|
||||
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
let reader =
|
||||
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
"app_reader_prepare_streaming",
|
||||
streaming_reader_stage_start,
|
||||
);
|
||||
reader
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_prepare", reader_prepare_stage_start);
|
||||
|
||||
if size >= 0 {
|
||||
let checksum_setup_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
if let Err(err) = reader.add_checksum(headers, trailer_source(trailing_headers.clone()), false) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
opts.want_checksum = reader.checksum();
|
||||
put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_reader_checksum_setup", checksum_setup_stage_start);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_path(put_path);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("ingress_prepare", ingress_stage_start);
|
||||
|
||||
@@ -19,6 +19,7 @@ use hyper::{Request, Response};
|
||||
use pin_project_lite::pin_project;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
use tower::Service;
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
|
||||
@@ -35,6 +36,13 @@ pub(crate) fn is_grpc_request<B>(req: &Request<B>) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn rename_data_grpc_stage(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
)
|
||||
}
|
||||
|
||||
/// The service that can serve both gRPC and REST HTTP Requests
|
||||
#[derive(Clone)]
|
||||
pub struct HybridService<Rest, Grpc> {
|
||||
@@ -71,8 +79,11 @@ where
|
||||
/// as a REST request
|
||||
fn call(&mut self, req: Request<Incoming>) -> Self::Future {
|
||||
if is_grpc_request(&req) {
|
||||
let observe_rename_data = rename_data_grpc_stage(req.uri().path());
|
||||
HybridFuture::Grpc {
|
||||
grpc_future: self.grpc.call(req),
|
||||
observe_rename_data,
|
||||
service_future_started: rustfs_io_metrics::put_stage_timer(),
|
||||
}
|
||||
} else {
|
||||
HybridFuture::Rest {
|
||||
@@ -93,7 +104,10 @@ pin_project! {
|
||||
},
|
||||
Grpc {
|
||||
#[pin]
|
||||
grpc_body: GrpcBody
|
||||
grpc_body: GrpcBody,
|
||||
first_frame_started: Option<Instant>,
|
||||
body_started: Option<Instant>,
|
||||
first_frame_recorded: bool,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -123,21 +137,44 @@ where
|
||||
fn is_end_stream(&self) -> bool {
|
||||
match self {
|
||||
Self::Rest { rest_body } => rest_body.is_end_stream(),
|
||||
Self::Grpc { grpc_body } => grpc_body.is_end_stream(),
|
||||
Self::Grpc { grpc_body, .. } => grpc_body.is_end_stream(),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
match self.project() {
|
||||
HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into),
|
||||
HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into),
|
||||
HybridBodyProj::Grpc {
|
||||
mut grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => {
|
||||
let poll = grpc_body.as_mut().poll_frame(cx).map_err(Into::into);
|
||||
if !*first_frame_recorded && let Poll::Ready(_) = &poll {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_FIRST_FRAME,
|
||||
first_frame_started.take(),
|
||||
);
|
||||
*first_frame_recorded = true;
|
||||
}
|
||||
let body_complete = matches!(&poll, Poll::Ready(None))
|
||||
|| (matches!(&poll, Poll::Ready(Some(Ok(_)))) && grpc_body.as_ref().get_ref().is_end_stream());
|
||||
if body_complete {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_BODY_COMPLETE,
|
||||
body_started.take(),
|
||||
);
|
||||
}
|
||||
poll
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
match self {
|
||||
Self::Rest { rest_body } => rest_body.size_hint(),
|
||||
Self::Grpc { grpc_body } => grpc_body.size_hint(),
|
||||
Self::Grpc { grpc_body, .. } => grpc_body.size_hint(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -154,6 +191,8 @@ pin_project! {
|
||||
Grpc {
|
||||
#[pin]
|
||||
grpc_future: GrpcFuture,
|
||||
observe_rename_data: bool,
|
||||
service_future_started: Option<Instant>,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -174,8 +213,27 @@ where
|
||||
Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
|
||||
Poll::Pending => Poll::Pending,
|
||||
},
|
||||
HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) {
|
||||
Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))),
|
||||
HybridFutureProj::Grpc {
|
||||
grpc_future,
|
||||
observe_rename_data,
|
||||
service_future_started,
|
||||
} => match grpc_future.poll(cx) {
|
||||
Poll::Ready(Ok(res)) => {
|
||||
if *observe_rename_data {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_TRANSPORT_SERVICE_FUTURE,
|
||||
service_future_started.take(),
|
||||
);
|
||||
}
|
||||
let first_frame_started = (*observe_rename_data).then(Instant::now);
|
||||
let body_started = (*observe_rename_data).then(Instant::now);
|
||||
Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded: !*observe_rename_data,
|
||||
})))
|
||||
}
|
||||
Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
|
||||
Poll::Pending => Poll::Pending,
|
||||
},
|
||||
|
||||
@@ -861,7 +861,20 @@ where
|
||||
}
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1010,7 +1023,20 @@ where
|
||||
}
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1161,7 +1187,20 @@ where
|
||||
|
||||
Response::from_parts(parts, HybridBody::Rest { rest_body })
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1238,7 +1277,20 @@ where
|
||||
},
|
||||
)
|
||||
}
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
@@ -1306,7 +1358,20 @@ where
|
||||
rest_body: RestBody::from(Bytes::new()),
|
||||
},
|
||||
),
|
||||
HybridBody::Grpc { grpc_body } => Response::from_parts(parts, HybridBody::Grpc { grpc_body }),
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
} => Response::from_parts(
|
||||
parts,
|
||||
HybridBody::Grpc {
|
||||
grpc_body,
|
||||
first_frame_started,
|
||||
body_started,
|
||||
first_frame_recorded,
|
||||
},
|
||||
),
|
||||
};
|
||||
|
||||
Ok(response)
|
||||
|
||||
@@ -1337,6 +1337,8 @@ impl NodeService {
|
||||
&self,
|
||||
request: Request<RenameDataRequest>,
|
||||
) -> Result<Response<RenameDataResponse>, Status> {
|
||||
let service_return_started = rustfs_io_metrics::put_stage_timer();
|
||||
let service_entry_started = rustfs_io_metrics::put_stage_timer();
|
||||
if !request.get_ref().scanner_publication_lease_token.is_empty() || !request.get_ref().bucket_incarnation_id.is_empty() {
|
||||
let has_body_digest = request
|
||||
.metadata()
|
||||
@@ -1344,14 +1346,37 @@ impl NodeService {
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value != "UNSIGNED-PAYLOAD");
|
||||
if !has_body_digest {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
service_entry_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(Status::permission_denied("scanner publication lease rename requires a body-bound digest"));
|
||||
}
|
||||
}
|
||||
verify_disk_mutation_digest(
|
||||
if let Err(err) = verify_disk_mutation_digest(
|
||||
&request,
|
||||
rustfs_protos::canonical_rename_data_request_body(request.get_ref()),
|
||||
"rename_data",
|
||||
)?;
|
||||
) {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
service_entry_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_ENTRY,
|
||||
service_entry_started,
|
||||
);
|
||||
let before_handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
let request = request.into_inner();
|
||||
let target = self.local_mutation_target();
|
||||
let bucket_incarnation = if request.bucket_incarnation_id.is_empty() {
|
||||
@@ -1362,6 +1387,14 @@ impl NodeService {
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| Status::invalid_argument("bucket incarnation must be a non-nil UUID"))?;
|
||||
if !request.scanner_publication_lease_token.is_empty() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(Status::invalid_argument(
|
||||
"heal incarnation and scanner publication lease cannot be combined",
|
||||
));
|
||||
@@ -1373,12 +1406,26 @@ impl NodeService {
|
||||
let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) {
|
||||
Ok(file_info) => file_info,
|
||||
Err(err) => {
|
||||
return Ok(Response::new(RenameDataResponse {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
let response_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
|
||||
}));
|
||||
});
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
|
||||
response_build_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
};
|
||||
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
|
||||
@@ -1387,6 +1434,14 @@ impl NodeService {
|
||||
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
|
||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
||||
if token.is_nil() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
|
||||
}
|
||||
Some(token)
|
||||
@@ -1394,6 +1449,11 @@ impl NodeService {
|
||||
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let observation = startup_cas_rename_observation(&target, &request, &decoded_file_info.value);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_BEFORE_HANDLER,
|
||||
before_handler_started,
|
||||
);
|
||||
let handler_await_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = target
|
||||
.rename_local_data(
|
||||
&request.disk,
|
||||
@@ -1404,6 +1464,11 @@ impl NodeService {
|
||||
bucket_incarnation,
|
||||
)
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_HANDLER_AWAIT,
|
||||
handler_await_started,
|
||||
);
|
||||
let after_handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(mut observation) = observation {
|
||||
observation["ok"] = serde_json::json!(result.is_ok());
|
||||
@@ -1411,28 +1476,47 @@ impl NodeService {
|
||||
let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
|
||||
let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
|
||||
}
|
||||
match result {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_AFTER_HANDLER,
|
||||
after_handler_started,
|
||||
);
|
||||
let response_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_encode_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = match result {
|
||||
Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) {
|
||||
Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse {
|
||||
Ok((rename_data_resp, rename_data_resp_bin)) => Response::new(RenameDataResponse {
|
||||
success: true,
|
||||
rename_data_resp,
|
||||
rename_data_resp_bin: rename_data_resp_bin.into(),
|
||||
error: None,
|
||||
})),
|
||||
Err(err) => Ok(Response::new(RenameDataResponse {
|
||||
}),
|
||||
Err(err) => Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}),
|
||||
},
|
||||
Err(err) => Ok(Response::new(RenameDataResponse {
|
||||
Err(err) => Response::new(RenameDataResponse {
|
||||
success: false,
|
||||
rename_data_resp: String::new(),
|
||||
rename_data_resp_bin: Vec::new().into(),
|
||||
error: Some(err.into()),
|
||||
})),
|
||||
}
|
||||
}),
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_ENCODE,
|
||||
response_encode_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_BUILD,
|
||||
response_build_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_SERVICE_RESPONSE_RETURN,
|
||||
service_return_started,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
pub(super) async fn handle_list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {
|
||||
|
||||
Reference in New Issue
Block a user