diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index f0d9e4712..8ebe17658 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -57,6 +58,7 @@ impl WarmBackendAliyun { return Err(std::io::Error::other(e.to_string())); } }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -151,3 +153,26 @@ fn optimal_part_size(object_size: i64) -> Result { } Ok(part_size) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::tier::tier_config::TierAliyun; + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = TierAliyun { + endpoint: "https://127.0.0.1:9000".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + match WarmBackendAliyun::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } +} diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index 4779e6fa5..997492ffb 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -57,6 +58,7 @@ impl WarmBackendAzure { return Err(std::io::Error::other(e.to_string())); } }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -151,3 +153,26 @@ fn optimal_part_size(object_size: i64) -> Result { } Ok(part_size) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::tier::tier_config::TierAzure; + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = TierAzure { + endpoint: "https://127.0.0.1:9000".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + match WarmBackendAzure::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } +} diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index adf6bf11b..63c9a62c1 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -39,6 +39,7 @@ use rustfs_s3_client::{ api_put_object::PutObjectOptions, transition_api::{Options, ReadCloser, ReaderImpl}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; @@ -73,6 +74,12 @@ impl WarmBackendGCS { return Err(std::io::Error::other("no bucket name was provided")); } + if !conf.endpoint.is_empty() { + let endpoint_url = url::Url::parse(&conf.endpoint).map_err(|e| std::io::Error::other(e.to_string()))?; + validate_outbound_url(&endpoint_url) + .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + } + let authorized_user = serde_json::from_str(&conf.creds)?; let credentials = Builder::new(authorized_user) //.with_retry_policy(AlwaysRetry.with_attempt_limit(3)) @@ -211,7 +218,9 @@ impl WarmBackend for WarmBackendGCS { #[cfg(test)] mod tests { + use super::WarmBackendGCS; use super::parse_generation; + use crate::services::tier::tier_config::TierGCS; use std::io::ErrorKind; #[test] @@ -231,6 +240,21 @@ mod tests { assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}"); } } + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_credential_setup() { + let conf = TierGCS { + endpoint: "https://127.0.0.1:9000".to_string(), + creds: "not-json".to_string(), + bucket: "tier-bucket".to_string(), + ..Default::default() + }; + + match WarmBackendGCS::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed"), "unexpected error: {err}"), + } + } } /*fn gcs_to_object_error(err: Error, params: Vec) -> Option { diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index 69a176861..33a506e73 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -57,6 +58,7 @@ impl WarmBackendHuaweicloud { return Err(std::io::Error::other(e.to_string())); } }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -152,3 +154,26 @@ fn optimal_part_size(object_size: i64) -> Result { } Ok(part_size) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::tier::tier_config::TierHuaweicloud; + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = TierHuaweicloud { + endpoint: "https://127.0.0.1:9000".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + match WarmBackendHuaweicloud::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } +} diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index 0ba037532..6051ff0bc 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -57,6 +58,7 @@ impl WarmBackendMinIO { return Err(std::io::Error::other(e.to_string())); } }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -168,3 +170,26 @@ fn optimal_part_size(object_size: i64) -> Result { } Ok(part_size) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::tier::tier_config::TierMinIO; + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = TierMinIO { + endpoint: "https://127.0.0.1:9000".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + match WarmBackendMinIO::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } +} diff --git a/crates/ecstore/src/services/tier/warm_backend_r2.rs b/crates/ecstore/src/services/tier/warm_backend_r2.rs index 9784537ba..d5fe9a101 100644 --- a/crates/ecstore/src/services/tier/warm_backend_r2.rs +++ b/crates/ecstore/src/services/tier/warm_backend_r2.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -57,6 +58,7 @@ impl WarmBackendR2 { return Err(std::io::Error::other(e.to_string())); } }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -168,3 +170,26 @@ fn optimal_part_size(object_size: i64) -> Result { } Ok(part_size) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::tier::tier_config::TierR2; + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = TierR2 { + endpoint: "https://127.0.0.1:9000".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + match WarmBackendR2::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } +} diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 33a10e93b..e11377ea6 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; const MAX_PARTS_COUNT: i64 = 10000; @@ -54,6 +55,7 @@ impl WarmBackendRustFS { Ok(u) => u, Err(e) => return Err(std::io::Error::other(e)), }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -196,4 +198,14 @@ mod tests { }; assert!(err.to_string().contains("host"), "expected host validation error, got: {err}"); } + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = rustfs_tier("https://127.0.0.1:9000"); + + match WarmBackendRustFS::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } } diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index 98376b68d..14781af90 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -32,6 +32,7 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; +use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -57,6 +58,7 @@ impl WarmBackendTencent { return Err(std::io::Error::other(e.to_string())); } }; + validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -151,3 +153,26 @@ fn optimal_part_size(object_size: i64) -> Result { } Ok(part_size) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::tier::tier_config::TierTencent; + + #[tokio::test] + async fn new_rejects_loopback_endpoint_before_network_setup() { + let conf = TierTencent { + endpoint: "https://127.0.0.1:9000".to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + + match WarmBackendTencent::new(&conf, "tier").await { + Ok(_) => panic!("loopback endpoint should be rejected"), + Err(err) => assert!(err.to_string().contains("not allowed")), + } + } +} diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 10e8b5f79..328e855c3 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -54,8 +54,15 @@ 19|crates/ecstore/src/services/rebalance/worker.rs 33|crates/ecstore/src/services/tier/tier.rs 1|crates/ecstore/src/services/tier/tier_config.rs -1|crates/ecstore/src/services/tier/warm_backend_gcs.rs +1|crates/ecstore/src/services/tier/warm_backend_aliyun.rs +1|crates/ecstore/src/services/tier/warm_backend_azure.rs +2|crates/ecstore/src/services/tier/warm_backend_gcs.rs +1|crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +1|crates/ecstore/src/services/tier/warm_backend_minio.rs +1|crates/ecstore/src/services/tier/warm_backend_r2.rs +1|crates/ecstore/src/services/tier/warm_backend_rustfs.rs 1|crates/ecstore/src/services/tier/warm_backend_s3.rs +1|crates/ecstore/src/services/tier/warm_backend_tencent.rs 1|crates/ecstore/src/services/tier/warm_backend_wasabi.rs 7|crates/ecstore/src/set_disk/core/io_primitives.rs 1|crates/ecstore/src/set_disk/mod.rs