diff --git a/crates/e2e_test/src/reliant/tiering.rs b/crates/e2e_test/src/reliant/tiering.rs index 401e3752c..0503ea9c7 100644 --- a/crates/e2e_test/src/reliant/tiering.rs +++ b/crates/e2e_test/src/reliant/tiering.rs @@ -23,9 +23,14 @@ //! //! There are no containers, no external S3 backend and no `awscurl`: the //! `AddTier` admin call is signed in-process with `rustfs_signer`, exactly like -//! the other admin-API e2e suites in this crate. The RustFS warm backend has no -//! loopback/SSRF restriction (that guard is replication-only), so `hot` can tier -//! to `cold` over `http://127.0.0.1:`. +//! the other admin-API e2e suites in this crate. Every warm backend's endpoint +//! (including RustFS) runs through the shared outbound policy +//! (crates/utils/src/egress.rs), which rejects loopback hosts by default, so +//! `hot` is started with `RUSTFS_OUTBOUND_ALLOW_ORIGINS` set to `cold`'s exact +//! origin (see `hot_env_for_tier`) to allow this hermetic suite's real +//! `http://127.0.0.1:` connectivity — the same operator escape hatch +//! already used for webhook targets and OIDC discovery URLs, not a relaxation +//! of the check itself. //! //! The hermetic tests drive the transition and restore paths and pin the //! chains required by ilm-7 and the restore follow-up: @@ -168,6 +173,28 @@ async fn signed_admin_request( Ok((status, text)) } +/// Extra child-process env for `hot` when it will be wired to a `cold` tier +/// target over loopback. +/// +/// `WarmBackendRustFS::new` now runs every tier endpoint through the shared +/// outbound policy (crates/utils/src/egress.rs), which rejects loopback hosts +/// by default just like the S3/Wasabi tier types already did. This hermetic +/// suite's `cold` target is a second embedded server on `127.0.0.1`, so `hot` +/// needs an explicit, exact-origin allowlist entry to reach it — the same +/// operator escape hatch already used for webhook targets and OIDC discovery +/// URLs, not a relaxation of the check itself (metadata/link-local/unspecified +/// hosts stay forbidden even with this set). +/// +/// Takes `cold`'s origin as a plain `&str` (rather than `&RustFSTestEnvironment`) +/// so building this env list never holds a live borrow of `cold` itself — tests +/// that later call a `&mut cold` method (e.g. `stop_server`) can pass an owned +/// clone of `cold.url` instead. +fn hot_env_for_tier<'a>(cold_origin: &'a str, extra: &[(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> { + let mut env = vec![("RUSTFS_OUTBOUND_ALLOW_ORIGINS", cold_origin)]; + env.extend_from_slice(extra); + env +} + /// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`. /// /// No `force`, so the server runs the real in-use / connectivity probe against @@ -888,7 +915,7 @@ async fn test_hermetic_transition_main_path() -> TestResult { // Hot/source server. A 1s scanner cycle is a backstop; transition is // primarily driven immediately by the multipart completion path. let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1")]) + hot.start_rustfs_server_with_env(vec![], &hot_env_for_tier(cold.url.as_str(), &[("RUSTFS_SCANNER_CYCLE", "1")])) .await?; let hot_client = hot.create_s3_client(); @@ -987,8 +1014,11 @@ async fn test_hermetic_transition_restore_failure_expiry_and_retry() -> TestResu cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier(cold.url.as_str(), &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -1116,8 +1146,14 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult { cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25); @@ -1222,8 +1258,14 @@ async fn test_manual_transition_async_job_status_polling() -> TestResult { cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -1321,8 +1363,14 @@ async fn test_manual_transition_async_limit_reports_terminal_partial() -> TestRe cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -1485,11 +1533,14 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes let mut hot = RustFSTestEnvironment::new().await?; hot.start_rustfs_server_with_env( vec![], - &[ - ("RUSTFS_SCANNER_ENABLED", "false"), - ("RUSTFS_SCANNER_CYCLE", "3600"), - (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"), - ], + &hot_env_for_tier( + cold.url.as_str(), + &[ + ("RUSTFS_SCANNER_ENABLED", "false"), + ("RUSTFS_SCANNER_CYCLE", "3600"), + (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"), + ], + ), ) .await?; let hot_client = hot.create_s3_client(); @@ -1593,8 +1644,14 @@ async fn test_manual_transition_async_different_buckets_admit_concurrently() -> cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -1714,8 +1771,14 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() -> cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -1807,8 +1870,14 @@ async fn test_manual_transition_async_worker_failure_reports_terminal_partial() cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; cold.stop_server(); @@ -1902,11 +1971,14 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled() let mut hot = RustFSTestEnvironment::new().await?; hot.start_rustfs_server_with_env( vec![], - &[ - ("RUSTFS_SCANNER_ENABLED", "false"), - ("RUSTFS_SCANNER_CYCLE", "3600"), - (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"), - ], + &hot_env_for_tier( + cold.url.as_str(), + &[ + ("RUSTFS_SCANNER_ENABLED", "false"), + ("RUSTFS_SCANNER_CYCLE", "3600"), + (MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"), + ], + ), ) .await?; let hot_client = hot.create_s3_client(); @@ -1999,12 +2071,18 @@ async fn test_manual_transition_async_cancel_after_process_restart_recovers_term let cold_client = cold.create_s3_client(); cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; - let restart_env = [ - ("RUSTFS_SCANNER_ENABLED", "false"), - ("RUSTFS_SCANNER_CYCLE", "3600"), - ("RUSTFS_MAX_TRANSITION_WORKERS", "1"), - ("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"), - ]; + // Owned copy: `cold` is stopped (a `&mut cold` call) below, and + // `restart_env` must stay valid past that point for the later restart. + let cold_origin = cold.url.clone(); + let restart_env = hot_env_for_tier( + &cold_origin, + &[ + ("RUSTFS_SCANNER_ENABLED", "false"), + ("RUSTFS_SCANNER_CYCLE", "3600"), + ("RUSTFS_MAX_TRANSITION_WORKERS", "1"), + ("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"), + ], + ); let mut hot = RustFSTestEnvironment::new().await?; hot.start_rustfs_server_with_env(vec![], &restart_env).await?; let hot_client = hot.create_s3_client(); @@ -2158,8 +2236,14 @@ async fn test_manual_transition_run_contract_no_status_cancel_fields() -> TestRe cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -2197,8 +2281,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke cold_client.create_bucket().bucket(TIER_BUCKET).send().await?; let mut hot = RustFSTestEnvironment::new().await?; - hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.start_rustfs_server_with_env( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let hot_client = hot.create_s3_client(); add_rustfs_tier(&hot, &cold).await?; @@ -2238,8 +2328,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke "continuation token must not expose the raw object prefix: {continuation}" ); - hot.restart_server_preserving_data(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]) - .await?; + hot.restart_server_preserving_data( + vec![], + &hot_env_for_tier( + cold.url.as_str(), + &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")], + ), + ) + .await?; let second = manual_transition_run_with_max_and_continuation( &hot, @@ -2274,12 +2370,15 @@ async fn test_manual_transition_run_queue_pressure_partial() -> TestResult { let mut hot = RustFSTestEnvironment::new().await?; hot.start_rustfs_server_with_env( vec![], - &[ - ("RUSTFS_SCANNER_ENABLED", "false"), - ("RUSTFS_SCANNER_CYCLE", "3600"), - ("RUSTFS_MAX_TRANSITION_WORKERS", "1"), - ("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1"), - ], + &hot_env_for_tier( + cold.url.as_str(), + &[ + ("RUSTFS_SCANNER_ENABLED", "false"), + ("RUSTFS_SCANNER_CYCLE", "3600"), + ("RUSTFS_MAX_TRANSITION_WORKERS", "1"), + ("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1"), + ], + ), ) .await?; let hot_client = hot.create_s3_client(); diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 8e0a59546..3d987920f 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -41,6 +41,7 @@ use rustfs_s3_client::{ api_put_object::{AdvancedPutOptions, PutObjectOptions}, transition_api::{ReadCloser, ReaderImpl}, }; +use rustfs_utils::egress::OutboundPolicy; use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; @@ -53,11 +54,31 @@ use std::collections::HashMap; use time::OffsetDateTime; use time::format_description::well_known::{Rfc2822, Rfc3339}; use tracing::{info, warn}; +use url::Url; pub type WarmBackendImpl = Box; const PROBE_OBJECT: &str = "probeobject"; +/// Validates a warm-tier `endpoint` URL against the shared outbound policy +/// (crates/utils/src/egress.rs), the same fail-closed check already applied to +/// webhook targets and OIDC discovery URLs. By default this rejects loopback, +/// RFC1918/link-local, and known cloud metadata-service hosts; an operator can +/// allowlist one exact self-hosted origin via `RUSTFS_OUTBOUND_ALLOW_ORIGINS` +/// (metadata, link-local, and unspecified addresses can never be allowlisted). +/// +/// Every `WarmBackendXxx::new` constructor must call this immediately after +/// parsing `conf.endpoint` and before building any credentials or network +/// client, so a tier endpoint can never reach the network unvalidated +/// regardless of provider (rustfs/backlog#2039 adversarial-review finding). +pub(crate) fn validate_tier_endpoint_url(url: &Url) -> Result<(), std::io::Error> { + let policy = + OutboundPolicy::from_env_cached().map_err(|err| std::io::Error::other(format!("invalid outbound policy: {err}")))?; + policy + .validate_url(url) + .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}"))) +} + #[derive(Default)] pub struct WarmBackendGetOpts { pub start_offset: i64, diff --git a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs index 603e67d43..776c8eacd 100644 --- a/crates/ecstore/src/services/tier/warm_backend_aliyun.rs +++ b/crates/ecstore/src/services/tier/warm_backend_aliyun.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierAliyun, - warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url}, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +32,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -58,7 +57,7 @@ impl WarmBackendAliyun { return Err(std::io::Error::other(e.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), diff --git a/crates/ecstore/src/services/tier/warm_backend_azure.rs b/crates/ecstore/src/services/tier/warm_backend_azure.rs index 7cd8783dc..9e684ea7e 100644 --- a/crates/ecstore/src/services/tier/warm_backend_azure.rs +++ b/crates/ecstore/src/services/tier/warm_backend_azure.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierAzure, - warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url}, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +32,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -58,7 +57,7 @@ impl WarmBackendAzure { return Err(std::io::Error::other(e.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), diff --git a/crates/ecstore/src/services/tier/warm_backend_gcs.rs b/crates/ecstore/src/services/tier/warm_backend_gcs.rs index 63c9a62c1..5555c1708 100644 --- a/crates/ecstore/src/services/tier/warm_backend_gcs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_gcs.rs @@ -32,14 +32,13 @@ use std::convert::TryFrom; use crate::services::tier::{ tier_config::TierGCS, - warm_backend::{WarmBackend, WarmBackendGetOpts}, + warm_backend::{WarmBackend, WarmBackendGetOpts, validate_tier_endpoint_url}, }; use rustfs_s3_client::{ admin_handler_utils::AdminError, api_put_object::PutObjectOptions, transition_api::{Options, ReadCloser, ReaderImpl}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; @@ -76,8 +75,7 @@ impl WarmBackendGCS { if !conf.endpoint.is_empty() { let endpoint_url = url::Url::parse(&conf.endpoint).map_err(|e| std::io::Error::other(e.to_string()))?; - validate_outbound_url(&endpoint_url) - .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&endpoint_url)?; } let authorized_user = serde_json::from_str(&conf.creds)?; diff --git a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs index ec6f08ea6..da8b6099b 100644 --- a/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs +++ b/crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierHuaweicloud, - warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url}, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +32,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -58,7 +57,7 @@ impl WarmBackendHuaweicloud { return Err(std::io::Error::other(e.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index 00e3d6153..82625928a 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -23,7 +23,9 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierMinIO, - warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{ + TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url, + }, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +34,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -58,7 +59,7 @@ impl WarmBackendMinIO { return Err(std::io::Error::other(e.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), diff --git a/crates/ecstore/src/services/tier/warm_backend_r2.rs b/crates/ecstore/src/services/tier/warm_backend_r2.rs index 435d971c4..7ffcba719 100644 --- a/crates/ecstore/src/services/tier/warm_backend_r2.rs +++ b/crates/ecstore/src/services/tier/warm_backend_r2.rs @@ -23,7 +23,9 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierR2, - warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{ + TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url, + }, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +34,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -58,7 +59,7 @@ impl WarmBackendR2 { return Err(std::io::Error::other(e.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index cceaa0194..3a24b970b 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -23,7 +23,9 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierRustFS, - warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{ + TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url, + }, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +34,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; const MAX_PARTS_COUNT: i64 = 10000; @@ -55,7 +56,7 @@ impl WarmBackendRustFS { Ok(u) => u, Err(e) => return Err(std::io::Error::other(e)), }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), @@ -186,18 +187,25 @@ mod tests { } } + // `validate_tier_endpoint_url` now runs before this constructor's own + // `u.host_str()` check, and it only accepts a parsed URL once its scheme + // is http(s) and its host is non-empty (http/https can never parse with a + // missing host per the WHATWG URL spec), so that local host check is + // unreachable in practice. This regression instead pins the shared policy + // rejecting a non-http(s) endpoint scheme, which is the case that now + // actually exercises this path first. #[tokio::test] - async fn new_returns_error_when_endpoint_has_no_host() { + async fn new_rejects_endpoint_with_disallowed_scheme() { let conf = rustfs_tier("rustfs://"); let outcome = AssertUnwindSafe(WarmBackendRustFS::new(&conf, "tier")).catch_unwind().await; let result = outcome.expect("initialization should return an error instead of panicking"); let err = match result { - Ok(_) => panic!("endpoint without host must be rejected"), + Ok(_) => panic!("endpoint with a non-http(s) scheme must be rejected"), Err(err) => err, }; - assert!(err.to_string().contains("host"), "expected host validation error, got: {err}"); + assert!(err.to_string().contains("scheme"), "expected scheme validation error, got: {err}"); } #[tokio::test] diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 55a29418c..f97c652da 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -26,7 +26,7 @@ use crate::services::tier::{ tier_config::TierS3, warm_backend::{ TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts, - build_transition_put_options, + build_transition_put_options, validate_tier_endpoint_url, }, }; use http::HeaderMap; @@ -41,7 +41,6 @@ use rustfs_s3_client::{ transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore}, transition_api::{ReadCloser, ReaderImpl}, }; -use rustfs_utils::egress::validate_outbound_url; use rustfs_utils::path::SLASH_SEPARATOR; use s3s::dto::BucketVersioningStatus; @@ -90,7 +89,7 @@ impl WarmBackendS3 { return Err(std::io::Error::other(err.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != "" || conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == "" diff --git a/crates/ecstore/src/services/tier/warm_backend_tencent.rs b/crates/ecstore/src/services/tier/warm_backend_tencent.rs index 8cf68d457..9a12cd334 100644 --- a/crates/ecstore/src/services/tier/warm_backend_tencent.rs +++ b/crates/ecstore/src/services/tier/warm_backend_tencent.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use crate::services::tier::{ tier_config::TierTencent, - warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, + warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url}, warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::{ @@ -32,7 +32,6 @@ use rustfs_s3_client::{ credentials::{Credentials, SignatureType, Static, Value}, transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore}, }; -use rustfs_utils::egress::validate_outbound_url; use tracing::warn; const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5; @@ -58,7 +57,7 @@ impl WarmBackendTencent { return Err(std::io::Error::other(e.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_tier_endpoint_url(&u)?; let creds = Credentials::new(Static(Value { access_key_id: conf.access_key.clone(), diff --git a/scripts/error-other-format-baseline.txt b/scripts/error-other-format-baseline.txt index 328e855c3..d1ff874ed 100644 --- a/scripts/error-other-format-baseline.txt +++ b/scripts/error-other-format-baseline.txt @@ -54,15 +54,8 @@ 19|crates/ecstore/src/services/rebalance/worker.rs 33|crates/ecstore/src/services/tier/tier.rs 1|crates/ecstore/src/services/tier/tier_config.rs -1|crates/ecstore/src/services/tier/warm_backend_aliyun.rs -1|crates/ecstore/src/services/tier/warm_backend_azure.rs -2|crates/ecstore/src/services/tier/warm_backend_gcs.rs -1|crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs -1|crates/ecstore/src/services/tier/warm_backend_minio.rs -1|crates/ecstore/src/services/tier/warm_backend_r2.rs -1|crates/ecstore/src/services/tier/warm_backend_rustfs.rs -1|crates/ecstore/src/services/tier/warm_backend_s3.rs -1|crates/ecstore/src/services/tier/warm_backend_tencent.rs +2|crates/ecstore/src/services/tier/warm_backend.rs +1|crates/ecstore/src/services/tier/warm_backend_gcs.rs 1|crates/ecstore/src/services/tier/warm_backend_wasabi.rs 7|crates/ecstore/src/set_disk/core/io_primitives.rs 1|crates/ecstore/src/set_disk/mod.rs