mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 19:24:58 +08:00
fix(tier): validate outbound URLs for all warm backend providers (#6764)
WarmBackendS3::new already rejects loopback, private, link-local, and cloud metadata-service endpoints via validate_outbound_url, but the Aliyun, Azure, Huaweicloud, Tencent, MinIO, R2, RustFS, and GCS warm backend constructors built their transition clients directly from conf.endpoint without the same check. The endpoint comes from the AddTier admin API, gated only by SetTierAction, which can be a narrower IAM grant than root. Any principal holding it could point one of these eight tier types at an internal address (loopback, RFC1918, link-local, or a cloud metadata IP) and have the server issue authenticated outbound requests to it, a server-side SSRF vector that the S3 and Wasabi tier types were already closed against. Apply the same validate_outbound_url check at construction time for all eight providers, before any credentials or network client are built, mirroring the existing WarmBackendS3 pattern. GCS keeps its default-endpoint behavior when conf.endpoint is empty and only validates an explicitly configured endpoint. Add a regression test per provider asserting that a loopback endpoint is rejected before any backend/network setup, matching the existing WarmBackendS3 coverage. Update the error(format!) ratchet baseline: these are one-shot admin tier-configuration validation errors returned once per AddTier call, not per-disk I/O errors that flow through reduce_errs quorum aggregation (backlog#1845), so the new ::other(format!) call sites do not introduce a quorum-bucketing hazard. They mirror the pre-existing, already-baselined warm_backend_s3.rs call site.
This commit is contained in:
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -57,6 +58,7 @@ impl WarmBackendAliyun {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -151,3 +153,26 @@ fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierAliyun;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierAliyun {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendAliyun::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -57,6 +58,7 @@ impl WarmBackendAzure {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -151,3 +153,26 @@ fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierAzure;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierAzure {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendAzure::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ use rustfs_s3_client::{
|
||||
api_put_object::PutObjectOptions,
|
||||
transition_api::{Options, ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
@@ -73,6 +74,12 @@ impl WarmBackendGCS {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
if !conf.endpoint.is_empty() {
|
||||
let endpoint_url = url::Url::parse(&conf.endpoint).map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
validate_outbound_url(&endpoint_url)
|
||||
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
}
|
||||
|
||||
let authorized_user = serde_json::from_str(&conf.creds)?;
|
||||
let credentials = Builder::new(authorized_user)
|
||||
//.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
|
||||
@@ -211,7 +218,9 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::WarmBackendGCS;
|
||||
use super::parse_generation;
|
||||
use crate::services::tier::tier_config::TierGCS;
|
||||
use std::io::ErrorKind;
|
||||
|
||||
#[test]
|
||||
@@ -231,6 +240,21 @@ mod tests {
|
||||
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_credential_setup() {
|
||||
let conf = TierGCS {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
creds: "not-json".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendGCS::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed"), "unexpected error: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -57,6 +58,7 @@ impl WarmBackendHuaweicloud {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -152,3 +154,26 @@ fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierHuaweicloud;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierHuaweicloud {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendHuaweicloud::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -57,6 +58,7 @@ impl WarmBackendMinIO {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -168,3 +170,26 @@ fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierMinIO;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierMinIO {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendMinIO::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -57,6 +58,7 @@ impl WarmBackendR2 {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -168,3 +170,26 @@ fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierR2;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierR2 {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendR2::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
@@ -54,6 +55,7 @@ impl WarmBackendRustFS {
|
||||
Ok(u) => u,
|
||||
Err(e) => return Err(std::io::Error::other(e)),
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -196,4 +198,14 @@ mod tests {
|
||||
};
|
||||
assert!(err.to_string().contains("host"), "expected host validation error, got: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = rustfs_tier("https://127.0.0.1:9000");
|
||||
|
||||
match WarmBackendRustFS::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -57,6 +58,7 @@ impl WarmBackendTencent {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -151,3 +153,26 @@ fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
|
||||
}
|
||||
Ok(part_size)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::tier::tier_config::TierTencent;
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_rejects_loopback_endpoint_before_network_setup() {
|
||||
let conf = TierTencent {
|
||||
endpoint: "https://127.0.0.1:9000".to_string(),
|
||||
bucket: "tier-bucket".to_string(),
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match WarmBackendTencent::new(&conf, "tier").await {
|
||||
Ok(_) => panic!("loopback endpoint should be rejected"),
|
||||
Err(err) => assert!(err.to_string().contains("not allowed")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,8 +54,15 @@
|
||||
19|crates/ecstore/src/services/rebalance/worker.rs
|
||||
33|crates/ecstore/src/services/tier/tier.rs
|
||||
1|crates/ecstore/src/services/tier/tier_config.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_gcs.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_aliyun.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_azure.rs
|
||||
2|crates/ecstore/src/services/tier/warm_backend_gcs.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_minio.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_r2.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_rustfs.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_s3.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_tencent.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_wasabi.rs
|
||||
7|crates/ecstore/src/set_disk/core/io_primitives.rs
|
||||
1|crates/ecstore/src/set_disk/mod.rs
|
||||
|
||||
Reference in New Issue
Block a user