mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 03:45:46 +08:00
fix(tier): switch outbound URL check to the operator-overridable policy
The initial fix (routing all warm-tier constructors through validate_outbound_url) rejected the hermetic reliant::tiering e2e suite's real hot->cold connection over 127.0.0.1, since two embedded RustFS servers in that suite talk to each other over loopback by design. validate_outbound_url has no override; OutboundPolicy (already used by webhook targets and OIDC discovery URLs) enforces the identical default restrictions but lets an operator allowlist one exact origin via RUSTFS_OUTBOUND_ALLOW_ORIGINS -- metadata, link-local, and unspecified addresses can never be allowlisted, so this does not reopen the SSRF gap the previous commit closed. Switch every warm-tier constructor (including S3, which folds Wasabi in via new_with_bucket_lookup) to this policy through one shared crates/ecstore/src/services/tier/ warm_backend.rs::validate_tier_endpoint_url helper, replacing the nine scattered validate_outbound_url call sites the previous commit added and consolidating their error(format!) ratchet accounting into one file. Update the e2e suite to set RUSTFS_OUTBOUND_ALLOW_ORIGINS to the cold node's real origin before starting/restarting hot, via a hot_env_for_tier helper, and fix the resulting borrow-checker conflict in the one test that stops cold mid-test by cloning its origin into an owned String first. Also retarget a WarmBackendRustFS unit test that asserted on a now-unreachable local host-missing message: the shared policy's http(s)-only scheme check runs first and is now what actually rejects that fixture's non-http endpoint. Impact: operators with an existing self-hosted RustFS/MinIO/etc. tier whose endpoint is a bare loopback/private/link-local IP literal (not a hostname) need RUSTFS_OUTBOUND_ALLOW_ORIGINS=<origin> set and the server restarted to keep that tier working after this change.
This commit is contained in:
@@ -23,9 +23,14 @@
|
||||
//!
|
||||
//! There are no containers, no external S3 backend and no `awscurl`: the
|
||||
//! `AddTier` admin call is signed in-process with `rustfs_signer`, exactly like
|
||||
//! the other admin-API e2e suites in this crate. The RustFS warm backend has no
|
||||
//! loopback/SSRF restriction (that guard is replication-only), so `hot` can tier
|
||||
//! to `cold` over `http://127.0.0.1:<port>`.
|
||||
//! the other admin-API e2e suites in this crate. Every warm backend's endpoint
|
||||
//! (including RustFS) runs through the shared outbound policy
|
||||
//! (crates/utils/src/egress.rs), which rejects loopback hosts by default, so
|
||||
//! `hot` is started with `RUSTFS_OUTBOUND_ALLOW_ORIGINS` set to `cold`'s exact
|
||||
//! origin (see `hot_env_for_tier`) to allow this hermetic suite's real
|
||||
//! `http://127.0.0.1:<port>` connectivity — the same operator escape hatch
|
||||
//! already used for webhook targets and OIDC discovery URLs, not a relaxation
|
||||
//! of the check itself.
|
||||
//!
|
||||
//! The hermetic tests drive the transition and restore paths and pin the
|
||||
//! chains required by ilm-7 and the restore follow-up:
|
||||
@@ -168,6 +173,28 @@ async fn signed_admin_request(
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
/// Extra child-process env for `hot` when it will be wired to a `cold` tier
|
||||
/// target over loopback.
|
||||
///
|
||||
/// `WarmBackendRustFS::new` now runs every tier endpoint through the shared
|
||||
/// outbound policy (crates/utils/src/egress.rs), which rejects loopback hosts
|
||||
/// by default just like the S3/Wasabi tier types already did. This hermetic
|
||||
/// suite's `cold` target is a second embedded server on `127.0.0.1`, so `hot`
|
||||
/// needs an explicit, exact-origin allowlist entry to reach it — the same
|
||||
/// operator escape hatch already used for webhook targets and OIDC discovery
|
||||
/// URLs, not a relaxation of the check itself (metadata/link-local/unspecified
|
||||
/// hosts stay forbidden even with this set).
|
||||
///
|
||||
/// Takes `cold`'s origin as a plain `&str` (rather than `&RustFSTestEnvironment`)
|
||||
/// so building this env list never holds a live borrow of `cold` itself — tests
|
||||
/// that later call a `&mut cold` method (e.g. `stop_server`) can pass an owned
|
||||
/// clone of `cold.url` instead.
|
||||
fn hot_env_for_tier<'a>(cold_origin: &'a str, extra: &[(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> {
|
||||
let mut env = vec![("RUSTFS_OUTBOUND_ALLOW_ORIGINS", cold_origin)];
|
||||
env.extend_from_slice(extra);
|
||||
env
|
||||
}
|
||||
|
||||
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
|
||||
///
|
||||
/// No `force`, so the server runs the real in-use / connectivity probe against
|
||||
@@ -888,7 +915,7 @@ async fn test_hermetic_transition_main_path() -> TestResult {
|
||||
// Hot/source server. A 1s scanner cycle is a backstop; transition is
|
||||
// primarily driven immediately by the multipart completion path.
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1")])
|
||||
hot.start_rustfs_server_with_env(vec![], &hot_env_for_tier(cold.url.as_str(), &[("RUSTFS_SCANNER_CYCLE", "1")]))
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
|
||||
@@ -987,8 +1014,11 @@ async fn test_hermetic_transition_restore_failure_expiry_and_retry() -> TestResu
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(cold.url.as_str(), &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1116,8 +1146,14 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
@@ -1222,8 +1258,14 @@ async fn test_manual_transition_async_job_status_polling() -> TestResult {
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1321,8 +1363,14 @@ async fn test_manual_transition_async_limit_reports_terminal_partial() -> TestRe
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1485,11 +1533,14 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
@@ -1593,8 +1644,14 @@ async fn test_manual_transition_async_different_buckets_admit_concurrently() ->
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1714,8 +1771,14 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1807,8 +1870,14 @@ async fn test_manual_transition_async_worker_failure_reports_terminal_partial()
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
cold.stop_server();
|
||||
@@ -1902,11 +1971,14 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
@@ -1999,12 +2071,18 @@ async fn test_manual_transition_async_cancel_after_process_restart_recovers_term
|
||||
let cold_client = cold.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let restart_env = [
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"),
|
||||
];
|
||||
// Owned copy: `cold` is stopped (a `&mut cold` call) below, and
|
||||
// `restart_env` must stay valid past that point for the later restart.
|
||||
let cold_origin = cold.url.clone();
|
||||
let restart_env = hot_env_for_tier(
|
||||
&cold_origin,
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"),
|
||||
],
|
||||
);
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &restart_env).await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
@@ -2158,8 +2236,14 @@ async fn test_manual_transition_run_contract_no_status_cancel_fields() -> TestRe
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -2197,8 +2281,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -2238,8 +2328,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
"continuation token must not expose the raw object prefix: {continuation}"
|
||||
);
|
||||
|
||||
hot.restart_server_preserving_data(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
hot.restart_server_preserving_data(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let second = manual_transition_run_with_max_and_continuation(
|
||||
&hot,
|
||||
@@ -2274,12 +2370,15 @@ async fn test_manual_transition_run_queue_pressure_partial() -> TestResult {
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1"),
|
||||
],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1"),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
|
||||
@@ -41,6 +41,7 @@ use rustfs_s3_client::{
|
||||
api_put_object::{AdvancedPutOptions, PutObjectOptions},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::egress::OutboundPolicy;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
@@ -53,11 +54,31 @@ use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::{Rfc2822, Rfc3339};
|
||||
use tracing::{info, warn};
|
||||
use url::Url;
|
||||
|
||||
pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;
|
||||
|
||||
const PROBE_OBJECT: &str = "probeobject";
|
||||
|
||||
/// Validates a warm-tier `endpoint` URL against the shared outbound policy
|
||||
/// (crates/utils/src/egress.rs), the same fail-closed check already applied to
|
||||
/// webhook targets and OIDC discovery URLs. By default this rejects loopback,
|
||||
/// RFC1918/link-local, and known cloud metadata-service hosts; an operator can
|
||||
/// allowlist one exact self-hosted origin via `RUSTFS_OUTBOUND_ALLOW_ORIGINS`
|
||||
/// (metadata, link-local, and unspecified addresses can never be allowlisted).
|
||||
///
|
||||
/// Every `WarmBackendXxx::new` constructor must call this immediately after
|
||||
/// parsing `conf.endpoint` and before building any credentials or network
|
||||
/// client, so a tier endpoint can never reach the network unvalidated
|
||||
/// regardless of provider (rustfs/backlog#2039 adversarial-review finding).
|
||||
pub(crate) fn validate_tier_endpoint_url(url: &Url) -> Result<(), std::io::Error> {
|
||||
let policy =
|
||||
OutboundPolicy::from_env_cached().map_err(|err| std::io::Error::other(format!("invalid outbound policy: {err}")))?;
|
||||
policy
|
||||
.validate_url(url)
|
||||
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct WarmBackendGetOpts {
|
||||
pub start_offset: i64,
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierAliyun,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +32,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -58,7 +57,7 @@ impl WarmBackendAliyun {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierAzure,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +32,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -58,7 +57,7 @@ impl WarmBackendAzure {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
|
||||
@@ -32,14 +32,13 @@ use std::convert::TryFrom;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierGCS,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, validate_tier_endpoint_url},
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_put_object::PutObjectOptions,
|
||||
transition_api::{Options, ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
|
||||
@@ -76,8 +75,7 @@ impl WarmBackendGCS {
|
||||
|
||||
if !conf.endpoint.is_empty() {
|
||||
let endpoint_url = url::Url::parse(&conf.endpoint).map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
validate_outbound_url(&endpoint_url)
|
||||
.map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&endpoint_url)?;
|
||||
}
|
||||
|
||||
let authorized_user = serde_json::from_str(&conf.creds)?;
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierHuaweicloud,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +32,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -58,7 +57,7 @@ impl WarmBackendHuaweicloud {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
|
||||
@@ -23,7 +23,9 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierMinIO,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url,
|
||||
},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +34,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -58,7 +59,7 @@ impl WarmBackendMinIO {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
|
||||
@@ -23,7 +23,9 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierR2,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url,
|
||||
},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +34,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -58,7 +59,7 @@ impl WarmBackendR2 {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
|
||||
@@ -23,7 +23,9 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierRustFS,
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url,
|
||||
},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +34,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
const MAX_PARTS_COUNT: i64 = 10000;
|
||||
@@ -55,7 +56,7 @@ impl WarmBackendRustFS {
|
||||
Ok(u) => u,
|
||||
Err(e) => return Err(std::io::Error::other(e)),
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
@@ -186,18 +187,25 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// `validate_tier_endpoint_url` now runs before this constructor's own
|
||||
// `u.host_str()` check, and it only accepts a parsed URL once its scheme
|
||||
// is http(s) and its host is non-empty (http/https can never parse with a
|
||||
// missing host per the WHATWG URL spec), so that local host check is
|
||||
// unreachable in practice. This regression instead pins the shared policy
|
||||
// rejecting a non-http(s) endpoint scheme, which is the case that now
|
||||
// actually exercises this path first.
|
||||
#[tokio::test]
|
||||
async fn new_returns_error_when_endpoint_has_no_host() {
|
||||
async fn new_rejects_endpoint_with_disallowed_scheme() {
|
||||
let conf = rustfs_tier("rustfs://");
|
||||
|
||||
let outcome = AssertUnwindSafe(WarmBackendRustFS::new(&conf, "tier")).catch_unwind().await;
|
||||
|
||||
let result = outcome.expect("initialization should return an error instead of panicking");
|
||||
let err = match result {
|
||||
Ok(_) => panic!("endpoint without host must be rejected"),
|
||||
Ok(_) => panic!("endpoint with a non-http(s) scheme must be rejected"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(err.to_string().contains("host"), "expected host validation error, got: {err}");
|
||||
assert!(err.to_string().contains("scheme"), "expected scheme validation error, got: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options,
|
||||
build_transition_put_options, validate_tier_endpoint_url,
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
@@ -41,7 +41,6 @@ use rustfs_s3_client::{
|
||||
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use s3s::dto::BucketVersioningStatus;
|
||||
|
||||
@@ -90,7 +89,7 @@ impl WarmBackendS3 {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
if conf.aws_role_web_identity_token_file == "" && conf.aws_role_arn != ""
|
||||
|| conf.aws_role_web_identity_token_file != "" && conf.aws_role_arn == ""
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::sync::Arc;
|
||||
|
||||
use crate::services::tier::{
|
||||
tier_config::TierTencent,
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options, validate_tier_endpoint_url},
|
||||
warm_backend_s3::WarmBackendS3,
|
||||
};
|
||||
use rustfs_s3_client::{
|
||||
@@ -32,7 +32,6 @@ use rustfs_s3_client::{
|
||||
credentials::{Credentials, SignatureType, Static, Value},
|
||||
transition_api::{BucketLookupType, Options, ReadCloser, ReaderImpl, TransitionClient, TransitionCore},
|
||||
};
|
||||
use rustfs_utils::egress::validate_outbound_url;
|
||||
use tracing::warn;
|
||||
|
||||
const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
|
||||
@@ -58,7 +57,7 @@ impl WarmBackendTencent {
|
||||
return Err(std::io::Error::other(e.to_string()));
|
||||
}
|
||||
};
|
||||
validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
validate_tier_endpoint_url(&u)?;
|
||||
|
||||
let creds = Credentials::new(Static(Value {
|
||||
access_key_id: conf.access_key.clone(),
|
||||
|
||||
@@ -54,15 +54,8 @@
|
||||
19|crates/ecstore/src/services/rebalance/worker.rs
|
||||
33|crates/ecstore/src/services/tier/tier.rs
|
||||
1|crates/ecstore/src/services/tier/tier_config.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_aliyun.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_azure.rs
|
||||
2|crates/ecstore/src/services/tier/warm_backend_gcs.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_minio.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_r2.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_rustfs.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_s3.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_tencent.rs
|
||||
2|crates/ecstore/src/services/tier/warm_backend.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_gcs.rs
|
||||
1|crates/ecstore/src/services/tier/warm_backend_wasabi.rs
|
||||
7|crates/ecstore/src/set_disk/core/io_primitives.rs
|
||||
1|crates/ecstore/src/set_disk/mod.rs
|
||||
|
||||
Reference in New Issue
Block a user