mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 05:45:37 +08:00
Shard integration tests by test class instead of by node id
pytest-shard assigns each test to a shard by hashing its node id. Integration test classes boot a Galaxy instance in setUpClass, so scattering one class's tests across shards makes every shard pay that class's startup. On run 30195939009 that turned 269 classes into 515 instance launches, and 313 of those launches served a single test. Keep a class's tests together and pack whole classes into shards longest-first, weighted by durations recorded from that run. Balancing by measured cost also replaces the hash-random shard balance, which spread the four shards over 109-124 minutes. The plugin reuses pytest-shard's --num-shards/--shard-id options and unregisters it, so the workflow invocation is unchanged and other suites keep the existing behavior. Collecting the suite four ways confirms the shards still partition it exactly - 1472 items, 269 groups, none spanning two shards - which is what test/unit/test_shard.py guards.
This commit is contained in:
mvdbeek
@@ -0,0 +1,121 @@
|
||||
"""Distribute tests across CI shards without splitting up test classes.
|
||||
|
||||
An integration test class boots a Galaxy instance in ``setUpClass``, so a class split
|
||||
across shards makes every one of them pay that startup. Whole classes are kept together
|
||||
and packed into shards longest-first, weighted by recorded durations, which balances
|
||||
shards by cost rather than by name.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import (
|
||||
Any,
|
||||
NamedTuple,
|
||||
)
|
||||
|
||||
import pytest
|
||||
|
||||
DURATIONS_ENV_VAR = "GALAXY_TEST_SHARD_DURATIONS"
|
||||
# Per-group seconds measured from a CI run, as {"module::Class": seconds}. Only affects
|
||||
# balance, never correctness: unknown groups get DEFAULT_GROUP_SECONDS. Regenerate from a
|
||||
# full unsharded run with --structured_data_report_file, summing each test's setup, call
|
||||
# and teardown into its group_key(). GALAXY_TEST_SHARD_DURATIONS overrides the path.
|
||||
DEFAULT_DURATIONS_FILE = os.path.join(os.path.dirname(__file__), "shard_durations.json")
|
||||
# What to assume a group costs when we have no measurement for it. Integration classes
|
||||
# are startup-dominated, so an unknown group is much closer to one startup than to zero.
|
||||
DEFAULT_GROUP_SECONDS = 40.0
|
||||
|
||||
|
||||
class ShardGroup(NamedTuple):
|
||||
"""Tests that share a Galaxy instance, and therefore must share a shard."""
|
||||
|
||||
key: str
|
||||
items: list[Any]
|
||||
seconds: float
|
||||
|
||||
|
||||
def group_key(item) -> str:
|
||||
"""Identify the instance an item belongs to.
|
||||
|
||||
Class for class-based tests, module for those taking a module-scoped instance
|
||||
fixture, and the file for items with no module at all.
|
||||
"""
|
||||
module = getattr(item, "module", None)
|
||||
if module is None:
|
||||
return item.nodeid.split("::")[0]
|
||||
cls = getattr(item, "cls", None)
|
||||
if cls is not None:
|
||||
return f"{module.__name__}::{cls.__name__}"
|
||||
return module.__name__
|
||||
|
||||
|
||||
def load_durations(path: str | None = None) -> dict[str, float]:
|
||||
"""Load recorded per-group durations, falling back to an empty mapping."""
|
||||
path = path or os.environ.get(DURATIONS_ENV_VAR) or DEFAULT_DURATIONS_FILE
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return {key: float(seconds) for key, seconds in json.load(fh).items()}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def build_groups(items: list[Any], durations: dict[str, float]) -> list[ShardGroup]:
|
||||
grouped: dict[str, list[Any]] = defaultdict(list)
|
||||
for item in items:
|
||||
grouped[group_key(item)].append(item)
|
||||
return [
|
||||
ShardGroup(key, group_items, durations.get(key, DEFAULT_GROUP_SECONDS)) for key, group_items in grouped.items()
|
||||
]
|
||||
|
||||
|
||||
def assign_groups_to_shards(groups: list[ShardGroup], num_shards: int) -> list[list[ShardGroup]]:
|
||||
"""Pack groups into shards longest-first, always onto the shard with the least work.
|
||||
|
||||
Greedy bin-packing. Ties break on group key: every shard runs this independently and
|
||||
the assignments must agree, or tests get dropped or duplicated.
|
||||
"""
|
||||
shards: list[list[ShardGroup]] = [[] for _ in range(num_shards)]
|
||||
loads = [0.0] * num_shards
|
||||
for group in sorted(groups, key=lambda g: (-g.seconds, g.key)):
|
||||
target = min(range(num_shards), key=lambda index: (loads[index], index))
|
||||
shards[target].append(group)
|
||||
loads[target] += group.seconds
|
||||
return shards
|
||||
|
||||
|
||||
def select_shard(items: list[Any], shard_id: int, num_shards: int, durations: dict[str, float]) -> list[Any]:
|
||||
groups = build_groups(items, durations)
|
||||
assigned = assign_groups_to_shards(groups, num_shards)[shard_id]
|
||||
selected = []
|
||||
for group in sorted(assigned, key=lambda g: g.key):
|
||||
selected.extend(group.items)
|
||||
return selected
|
||||
|
||||
|
||||
def pytest_configure(config) -> None:
|
||||
"""Take over sharding from pytest-shard, reusing its command line options."""
|
||||
shard_plugin = config.pluginmanager.get_plugin("pytest-shard")
|
||||
if shard_plugin is not None:
|
||||
config.pluginmanager.unregister(shard_plugin)
|
||||
|
||||
|
||||
def pytest_report_collectionfinish(config, items) -> str | None:
|
||||
"""Report which groups this shard runs."""
|
||||
num_shards = config.getoption("num_shards", default=1)
|
||||
if num_shards <= 1:
|
||||
return None
|
||||
groups = sorted({group_key(item) for item in items})
|
||||
shard_id = config.getoption("shard_id", default=0)
|
||||
return f"Running {len(items)} items from {len(groups)} groups in shard {shard_id}: {', '.join(groups)}"
|
||||
|
||||
|
||||
@pytest.hookimpl(trylast=True)
|
||||
def pytest_collection_modifyitems(config, items: list[Any]) -> None:
|
||||
num_shards = config.getoption("num_shards", default=1)
|
||||
shard_id = config.getoption("shard_id", default=0)
|
||||
if num_shards <= 1:
|
||||
return
|
||||
if shard_id >= num_shards:
|
||||
raise ValueError(f"shard_id = {shard_id} must be less than num_shards = {num_shards}")
|
||||
items[:] = select_shard(items, shard_id, num_shards, load_durations())
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"integration.objectstore.test_azure": 10.9,
|
||||
"integration.objectstore.test_bulk_storage_operations::TestBulkStorageOperationsIntegration": 499.4,
|
||||
"integration.objectstore.test_changing_objectstore::TestChangingStoreObjectStoreIntegration": 42.7,
|
||||
"integration.objectstore.test_direct_download_redirect::TestDirectDownloadRedirectIntegration": 47.4,
|
||||
"integration.objectstore.test_jobs": 38.1,
|
||||
"integration.objectstore.test_jobs::TestObjectStoreJobsIntegration": 37.2,
|
||||
"integration.objectstore.test_mixed_store_by::TestMixedStoreByObjectStoreIntegration": 35.1,
|
||||
"integration.objectstore.test_objectstore_datatype_upload": 814.5,
|
||||
"integration.objectstore.test_onedata_objectstore": 447.7,
|
||||
"integration.objectstore.test_per_user::TestPerUserObjectStoreIntegration": 64.7,
|
||||
"integration.objectstore.test_per_user::TestPerUserObjectStoreQuotaIntegration": 52.0,
|
||||
"integration.objectstore.test_per_user::TestPerUserObjectStoreUpgradesIntegration": 31.0,
|
||||
"integration.objectstore.test_per_user::TestPerUserObjectStoreUpgradesWithSecretsIntegration": 72.0,
|
||||
"integration.objectstore.test_per_user::TestPerUserObjectStoreWithExtendedMetadataIntegration": 45.1,
|
||||
"integration.objectstore.test_per_user::TestPerUserObjectStoreWithSecretsIntegration": 68.5,
|
||||
"integration.objectstore.test_private_handling::TestPrivateCannotWritePublicDataObjectStoreIntegration": 30.1,
|
||||
"integration.objectstore.test_private_handling::TestPrivatePreventsSharingObjectStoreIntegration": 41.4,
|
||||
"integration.objectstore.test_private_handling_library_imports::TestToolOutputFromLibraryImportWithPrivateObjectStore": 42.9,
|
||||
"integration.objectstore.test_quota_limit::TestQuotaIntegration": 51.8,
|
||||
"integration.objectstore.test_remote_objectstore_cache_operations::TestCacheOperation": 39.4,
|
||||
"integration.objectstore.test_remote_objectstore_cache_operations::TestCacheOperationWithNoCacheUpdate": 34.7,
|
||||
"integration.objectstore.test_rucio_objectstore": 343.0,
|
||||
"integration.objectstore.test_selection_with_resource_parameters::TestObjectStoreSelectionWithResourceParameterIntegration": 34.9,
|
||||
"integration.objectstore.test_selection_with_user_preferred_object_store::TestObjectStoreSelectionWithExtendedMetadataIntegration": 322.9,
|
||||
"integration.objectstore.test_selection_with_user_preferred_object_store::TestObjectStoreSelectionWithPreferredObjectStoresIntegration": 130.3,
|
||||
"integration.objectstore.test_swift_objectstore": 212.9,
|
||||
"integration.oidc.test_auth_oidc::TestFixedDelegatedAuthIntegration": 77.1,
|
||||
"integration.oidc.test_auth_oidc::TestGalaxyOIDCLoginIntegration": 107.9,
|
||||
"integration.oidc.test_auth_oidc::TestWithoutFixedDelegatedAuth": 66.8,
|
||||
"integration.test_agents::TestAgentOperationsManagerEncoding": 29.3,
|
||||
"integration.test_agents::TestAgentsApi": 32.6,
|
||||
"integration.test_agents::TestAgentsApiLiveLLM": 0.1,
|
||||
"integration.test_agents::TestMCPServerSmoke": 66.7,
|
||||
"integration.test_apache_nginx_sendfile::TestApacheSendFileHeader": 46.1,
|
||||
"integration.test_apache_nginx_sendfile::TestNginxAccelHeader": 42.8,
|
||||
"integration.test_async_downloads::TestAsyncDownloadsIntegration": 29.7,
|
||||
"integration.test_celery_fetch_quota::TestCeleryFetchQuotaEnforcement": 36.9,
|
||||
"integration.test_celery_tasks::TestCeleryTasksIntegration": 42.9,
|
||||
"integration.test_celery_user_concurrency_limit::TestCeleryUserConcurrencyLimitDisabled": 30.5,
|
||||
"integration.test_celery_user_concurrency_limit::TestCeleryUserConcurrencyLimitPostgres": 47.9,
|
||||
"integration.test_celery_user_concurrency_limit::TestCeleryUserConcurrencyLimitSqlite": 37.6,
|
||||
"integration.test_celery_user_rate_limit::TestCeleryUserRateLimitIntegrationNoLimit": 27.9,
|
||||
"integration.test_celery_user_rate_limit::TestCeleryUserRateLimitIntegrationPostgres": 48.5,
|
||||
"integration.test_celery_user_rate_limit::TestCeleryUserRateLimitIntegrationSqlite": 51.6,
|
||||
"integration.test_chained_dynamic_destinations::TestChainedDynamicDestinationIntegration": 34.9,
|
||||
"integration.test_cli_runners::TestParamikoCliOpenPBSIntegration": 43.8,
|
||||
"integration.test_cli_runners::TestParamikoCliSlurmIntegration": 66.4,
|
||||
"integration.test_cli_runners::TestShellJobCliOpenPBSIntegration": 61.2,
|
||||
"integration.test_cli_runners::TestShellJobCliSlurmIntegration": 80.1,
|
||||
"integration.test_coexecution::TestCoexecution": 0.1,
|
||||
"integration.test_coexecution::TestKubernetesDependencyResolutionIntegration": 0.1,
|
||||
"integration.test_coexecution::TestKubernetesStagingContainerIntegration": 0.1,
|
||||
"integration.test_coexecution::TestTesCoexecutionContainerIntegration": 0.1,
|
||||
"integration.test_coexecution::TestTesCoexecutionCustomAmqpKeyContainerIntegration": 0.1,
|
||||
"integration.test_coexecution::TestTesDependencyResolutionIntegration": 0.1,
|
||||
"integration.test_config_options_users::TestDefaultUserExposeIntegration": 27.8,
|
||||
"integration.test_config_options_users::TestEmailUserExposeIntegration": 34.3,
|
||||
"integration.test_config_options_users::TestUsernameUserExposeIntegration": 32.5,
|
||||
"integration.test_config_schema::TestConfigSchema": 31.4,
|
||||
"integration.test_configuration_decode::TestConfigurationDecodeIntegration": 33.2,
|
||||
"integration.test_container_resolvers::TestCachedExplicitSingularityContainerResolver": 45.0,
|
||||
"integration.test_container_resolvers::TestCachedExplicitSingularityContainerResolverWithNamespace": 48.6,
|
||||
"integration.test_container_resolvers::TestCachedExplicitSingularityContainerResolverWithSingularityRequirement": 39.8,
|
||||
"integration.test_container_resolvers::TestCondaFallBack": 49.5,
|
||||
"integration.test_container_resolvers::TestCondaFallBackAndRequireContainer": 46.1,
|
||||
"integration.test_container_resolvers::TestDefaultContainerResolvers": 73.4,
|
||||
"integration.test_container_resolvers::TestDefaultSingularityContainerResolvers": 65.4,
|
||||
"integration.test_container_resolvers::TestExplicitContainerResolver": 34.1,
|
||||
"integration.test_container_resolvers::TestExplicitSingularityContainerResolver": 38.1,
|
||||
"integration.test_container_resolvers::TestMulledContainerResolvers": 46.2,
|
||||
"integration.test_container_resolvers::TestMulledContainerResolversNoAutoInstall": 42.3,
|
||||
"integration.test_container_resolvers::TestMulledSingularityContainerResolvers": 37.8,
|
||||
"integration.test_container_resolvers::TestMulledSingularityContainersResolversNoAutoInstall": 36.3,
|
||||
"integration.test_containerized_jobs::TestDockerizedJobsIntegration": 146.3,
|
||||
"integration.test_containerized_jobs::TestInlineContainerConfiguration": 30.1,
|
||||
"integration.test_containerized_jobs::TestInlineJobEnvironmentContainerResolver": 36.4,
|
||||
"integration.test_containerized_jobs::TestMappingContainerResolver": 37.9,
|
||||
"integration.test_containerized_jobs::TestPerDestinationContainerConfiguration": 36.7,
|
||||
"integration.test_containerized_jobs::TestSingularityJobsIntegration": 174.1,
|
||||
"integration.test_credentials::TestCredentialsApi": 39.9,
|
||||
"integration.test_data_manager::TestDataManagerIntegration": 73.9,
|
||||
"integration.test_data_manager_refgenie::TestDataManagerIntegration": 45.5,
|
||||
"integration.test_data_manager_workflow_bundle::TestDataManagerWorkflowInvocation": 68.9,
|
||||
"integration.test_dataset_copy_metadata_files::TestDirectoryStrategyMetadataFileIntegrationTestCase": 52.4,
|
||||
"integration.test_dataset_copy_metadata_files::TestExtendedMetadataStrategyMetadataFileIntegrationTestCase": 51.2,
|
||||
"integration.test_dataset_hashing::TestDatasetHashingAlwaysIntegration": 30.7,
|
||||
"integration.test_dataset_hashing::TestDatasetHashingIntegration": 38.6,
|
||||
"integration.test_dataset_hashing::TestDatasetHashingNeverIntegration": 34.3,
|
||||
"integration.test_dataset_hashing::TestDatasetHashingUploadIntegration": 35.2,
|
||||
"integration.test_datatype_upload": 589.0,
|
||||
"integration.test_default_permissions::TestDefaultPermissionsIntegration": 32.2,
|
||||
"integration.test_default_permissions::TestPrivateDefaultPermissionsIntegration": 28.8,
|
||||
"integration.test_default_permissions::TestPublicDefaultPermissionsIntegration": 37.7,
|
||||
"integration.test_drs_compact_identifiers::TestDRSCompactIdentifiersIntegration": 0.1,
|
||||
"integration.test_dynamic_edam_loading::TestDynamicEdamLoadingIntegration": 34.4,
|
||||
"integration.test_edam_toolbox::TestEdamToolboxDefaultIntegration": 29.5,
|
||||
"integration.test_edam_toolbox::TestEdamToolboxIntegration": 28.3,
|
||||
"integration.test_entry_point_sse::TestEntryPointSSEIntegration": 60.8,
|
||||
"integration.test_error_report::TestErrorEmailReportIntegration": 32.1,
|
||||
"integration.test_error_report::TestErrorReportIntegration": 35.4,
|
||||
"integration.test_event_loop_blocking::TestAiocopBlockingDetection": 31.0,
|
||||
"integration.test_extended_metadata": 895.6,
|
||||
"integration.test_extended_metadata::TestExtendedMetadataDeferredIntegration": 48.7,
|
||||
"integration.test_extended_metadata::TestExtendedMetadataIntegration": 169.7,
|
||||
"integration.test_extended_metadata_mapping::TestExtendedMetadataMappingIntegration": 56.0,
|
||||
"integration.test_extended_metadata_outputs_to_working_directory": 509.5,
|
||||
"integration.test_fail_job_tool_unavailable::TestFailJobWhenToolUnavailable": 49.3,
|
||||
"integration.test_flush_per_n_datasets": 41.5,
|
||||
"integration.test_galaxy_interactor::TestGalaxyInteractor": 29.8,
|
||||
"integration.test_genomes::TestGenomes": 31.0,
|
||||
"integration.test_handler_assignment_methods::TestDBPreassignHandlerAssignmentMethodIntegration": 36.6,
|
||||
"integration.test_handler_assignment_methods::TestDBSkipLockedHandlerAssignmentMethodIntegration": 28.6,
|
||||
"integration.test_handler_assignment_methods::TestDBTransactionIsolationHandlerAssignmentMethodIntegration": 38.4,
|
||||
"integration.test_hashicorp_vault::TestHashicorpVaultRenewalGalaxyIntegration": 38.2,
|
||||
"integration.test_history_archiving::TestHistoryArchivingAdmin": 75.4,
|
||||
"integration.test_history_archiving::TestHistoryArchivingWithExportRecord": 65.1,
|
||||
"integration.test_history_import_export::TestImportExportHistoryContentsViaTasksIntegration": 40.5,
|
||||
"integration.test_history_import_export::TestImportExportHistoryOutputsToWorkingDirIntegration": 215.6,
|
||||
"integration.test_history_import_export::TestImportExportHistoryViaTasksIntegration": 93.3,
|
||||
"integration.test_history_sse::TestHistorySSEIntegration": 44.9,
|
||||
"integration.test_htcondor_runner": 193.0,
|
||||
"integration.test_htcondor_runner::TestHTCondorContainerJob": 69.4,
|
||||
"integration.test_interactivetools_api::TestInteractiveToolsIntegration": 82.1,
|
||||
"integration.test_interactivetools_api::TestInteractiveToolsPulsarIntegration": 61.1,
|
||||
"integration.test_interactivetools_api::TestInteractiveToolsRemoteProxyIntegration": 0.9,
|
||||
"integration.test_interactivetools_api::TestInteractiveToolsShortURLIntegration": 59.0,
|
||||
"integration.test_interactivetools_api::TestKubeInteractiveToolsRemoteProxyIntegration": 0.1,
|
||||
"integration.test_job_cache::TestJobCacheFiltering": 32.9,
|
||||
"integration.test_job_environments::TestDefaultJobEnvironmentIntegration": 40.2,
|
||||
"integration.test_job_environments::TestEmbeddedPulsarDefaultJobEnvironmentIntegration": 44.5,
|
||||
"integration.test_job_environments::TestJobIOEnvironmentIntegration": 29.3,
|
||||
"integration.test_job_environments::TestSharedHomeJobEnvironmentIntegration": 41.0,
|
||||
"integration.test_job_environments::TestTmpDirAsShellCommandJobEnvironmentIntegration": 31.4,
|
||||
"integration.test_job_environments::TestTmpDirToTrueJobEnvironmentIntegration": 36.2,
|
||||
"integration.test_job_files::TestJobFilesIntegration": 33.1,
|
||||
"integration.test_job_files_tus": 48.9,
|
||||
"integration.test_job_outputs_to_working_directory": 39.0,
|
||||
"integration.test_job_recovery::TestJobRecoveryAfterHandledIntegration": 44.7,
|
||||
"integration.test_job_recovery::TestJobRecoveryBeforeHandledIntegration": 37.5,
|
||||
"integration.test_job_resource_error_recovery::TestJobRecoveryBeforeHandledIntegration": 29.4,
|
||||
"integration.test_job_resubmission::TestJobResubmissionDefaultIntegration": 39.3,
|
||||
"integration.test_job_resubmission::TestJobResubmissionDynamicIntegration": 29.5,
|
||||
"integration.test_job_resubmission::TestJobResubmissionDynamicMultipleIntegration": 28.3,
|
||||
"integration.test_job_resubmission::TestJobResubmissionIntegration": 138.1,
|
||||
"integration.test_job_resubmission::TestJobResubmissionPulsarIntegration": 39.3,
|
||||
"integration.test_job_resubmission::TestJobResubmissionSmallMemoryIntegration": 29.1,
|
||||
"integration.test_job_resubmission::TestJobResubmissionSmallMemoryResubmitsToLargeIntegration": 29.5,
|
||||
"integration.test_job_resubmission::TestJobResubmissionToolDetectedErrorIntegration": 38.0,
|
||||
"integration.test_job_resubmission::TestJobResubmissionToolDetectedErrorResubmitsIntegration": 30.1,
|
||||
"integration.test_kubernetes_runner::TestKubernetesIntegration": 185.3,
|
||||
"integration.test_landing_requests::TestLandingRequestsIntegration": 31.5,
|
||||
"integration.test_landing_requests::TestLandingRequestsWithoutHeadersConfigIntegration": 37.5,
|
||||
"integration.test_landing_requests::TestLandingRequestsWithoutVaultIntegration": 35.4,
|
||||
"integration.test_legacy_store_by": 47.4,
|
||||
"integration.test_legacy_store_by::TestChangeDatatypeStoreByIdIntegration": 46.1,
|
||||
"integration.test_live_evals::TestLiveEvals": 0.1,
|
||||
"integration.test_local_job_cancellation::TestLocalJobCancellation": 42.0,
|
||||
"integration.test_materialize_dataset_instance_tasks::TestMaterializeDatasetInstanceTasaksIntegration": 57.2,
|
||||
"integration.test_max_discovered_files::TestExtendedMetadataMaxDiscoveredFiles": 39.5,
|
||||
"integration.test_max_discovered_files::TestMaxDiscoveredFiles": 29.9,
|
||||
"integration.test_metadata_strategy::TestDiskUsageCeleryExtended": 40.8,
|
||||
"integration.test_metadata_strategy::TestDiskUsageUpdateDefault": 32.5,
|
||||
"integration.test_model_store_scripts::TestModelStoreScriptsIntegration": 30.7,
|
||||
"integration.test_notification_sse::TestNotificationSSEIntegration": 52.6,
|
||||
"integration.test_notifications::TestNotificationsIntegration": 39.0,
|
||||
"integration.test_notifications::TestNotificationsIntegrationTaskBased": 41.7,
|
||||
"integration.test_page_revision_json_encoding::TestPageJsonEncodingIntegration": 33.4,
|
||||
"integration.test_panel_views::TestPanelViewsFromConfigIntegration": 28.5,
|
||||
"integration.test_panel_views::TestPanelViewsFromDirectoryIntegration": 32.9,
|
||||
"integration.test_panel_views::TestPanelViewsWithShedTools": 41.1,
|
||||
"integration.test_plugins::TestPluginsInferenceServicesConfig": 26.2,
|
||||
"integration.test_plugins::TestPluginsInferenceServicesDefault": 26.5,
|
||||
"integration.test_plugins::TestVisualizationPluginsApi": 36.5,
|
||||
"integration.test_prefix_handling::TestPrefixUrlSerializationIntegration": 35.8,
|
||||
"integration.test_pulsar_embedded": 370.5,
|
||||
"integration.test_pulsar_embedded::TestEmbeddedPulsarIntegrationInstance": 70.0,
|
||||
"integration.test_pulsar_embedded_containers": 33.5,
|
||||
"integration.test_pulsar_embedded_containers::TestEmbeddedDockerPulsarIntegration": 35.8,
|
||||
"integration.test_pulsar_embedded_containers::TestEmbeddedSingularityPulsarIntegration": 44.2,
|
||||
"integration.test_pulsar_embedded_copy_working": 42.4,
|
||||
"integration.test_pulsar_embedded_extended_metadata": 95.1,
|
||||
"integration.test_pulsar_embedded_mq": 84.0,
|
||||
"integration.test_pulsar_embedded_mq::TestEmbeddedMessageQueuePulsarExtendedMetadataPurge": 42.5,
|
||||
"integration.test_pulsar_embedded_mq::TestEmbeddedMessageQueuePulsarPurge": 47.6,
|
||||
"integration.test_pulsar_embedded_none": 48.9,
|
||||
"integration.test_pulsar_embedded_relay": 74.4,
|
||||
"integration.test_pulsar_embedded_remote_metadata": 67.7,
|
||||
"integration.test_purge_datasets::TestPurgeDatasetsIntegration": 60.5,
|
||||
"integration.test_purge_datasets::TestPurgeDatasetsWithoutCeleryIntegration": 102.7,
|
||||
"integration.test_quota::TestQuotaIntegration": 34.7,
|
||||
"integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageHierarchicalIntegration": 31.3,
|
||||
"integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageHierarchicalNoTaskIntegration": 50.0,
|
||||
"integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageIntegration": 33.8,
|
||||
"integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageSimpleHierarchicalIntegration": 31.2,
|
||||
"integration.test_remote_files::TestRemoteFilesIntegration": 194.9,
|
||||
"integration.test_remote_files::TestRemoteFilesNotConfiguredIntegration": 29.1,
|
||||
"integration.test_remote_files_histories::TestRemoteFilesHistoryImportExportIntegration": 61.5,
|
||||
"integration.test_remote_files_posix::TestPosixFileSourceIntegration": 33.6,
|
||||
"integration.test_remote_files_posix::TestPreferLinksPosixFileSourceIntegration": 61.7,
|
||||
"integration.test_repository_operations::TestRepositoryInstallIntegrationTestCase": 71.1,
|
||||
"integration.test_resolvers::TestCondaResolutionIntegration": 115.4,
|
||||
"integration.test_save_job_id_on_datasets": 102.1,
|
||||
"integration.test_scripts::TestScriptsIntegration": 145.3,
|
||||
"integration.test_scripts_pgcleanup::TestPgCleanupUserObjectStoreIntegration": 50.7,
|
||||
"integration.test_scripts_pgcleanup::TestScriptsPgCleanupIntegration": 169.2,
|
||||
"integration.test_shed_tool_tests::TestToolShedToolTestIntegration": 59.1,
|
||||
"integration.test_storage_cleaner::TestStorageCleaner": 40.1,
|
||||
"integration.test_structured_dataset::TestStructuredDataset": 46.3,
|
||||
"integration.test_structured_like_unpopulated::TestStructuredLikeUnpopulatedRaisesNotReady": 29.4,
|
||||
"integration.test_tool_data_bundles::TestDataBundlesIntegration": 62.4,
|
||||
"integration.test_tool_data_delete::TestAdminToolDataIntegration": 47.7,
|
||||
"integration.test_tool_submission_errors::TestFailJobWhenToolUnavailable": 38.8,
|
||||
"integration.test_upload_configuration_options::TestAdminsCanPasteFilePaths": 47.4,
|
||||
"integration.test_upload_configuration_options::TestAdvancedFtpUploadFetch": 29.5,
|
||||
"integration.test_upload_configuration_options::TestAutoDecompress": 35.3,
|
||||
"integration.test_upload_configuration_options::TestDefaultBinaryContentFilters": 29.6,
|
||||
"integration.test_upload_configuration_options::TestDirectoryAndCompressedTypes": 33.4,
|
||||
"integration.test_upload_configuration_options::TestDisableContentChecking": 30.4,
|
||||
"integration.test_upload_configuration_options::TestDisableFtpPurgeUploadConfiguration": 30.4,
|
||||
"integration.test_upload_configuration_options::TestEnableFtpPurgeUploadConfiguration": 30.4,
|
||||
"integration.test_upload_configuration_options::TestExplicitEmailAsIdentifierFtpUploadConfiguration": 40.7,
|
||||
"integration.test_upload_configuration_options::TestFetchByPath": 47.8,
|
||||
"integration.test_upload_configuration_options::TestInvalidFetchRequests": 29.5,
|
||||
"integration.test_upload_configuration_options::TestLinkDataUploadExtendedMetadata": 34.2,
|
||||
"integration.test_upload_configuration_options::TestLocalAddressWhitelisting": 29.2,
|
||||
"integration.test_upload_configuration_options::TestNonAdminsCannotPasteFilePath": 30.4,
|
||||
"integration.test_upload_configuration_options::TestPerUsernameFtpUploadConfiguration": 35.1,
|
||||
"integration.test_upload_configuration_options::TestServerDirectoryOffByDefault": 26.4,
|
||||
"integration.test_upload_configuration_options::TestServerDirectoryValidUsage": 40.2,
|
||||
"integration.test_upload_configuration_options::TestSimpleFtpUploadConfiguration": 29.3,
|
||||
"integration.test_upload_configuration_options::TestTemplatedFtpDirectoryUploadConfiguration": 30.2,
|
||||
"integration.test_upload_configuration_options::TestUploadOptionsFtpUploadConfiguration": 52.2,
|
||||
"integration.test_upload_configuration_options::TestUploadWithDirectoryMetadata": 30.6,
|
||||
"integration.test_upload_configuration_options::TestUploadWithExtendedMetadata": 29.3,
|
||||
"integration.test_upload_configuration_options::TestUserServerDirectoryOffByDefault": 26.2,
|
||||
"integration.test_upload_configuration_options::TestUserServerDirectoryValidUsage": 42.5,
|
||||
"integration.test_user_defined_tool_job_conf::TestUserDefinedToolRecommendedJobSetup": 49.4,
|
||||
"integration.test_user_defined_tool_job_conf::TestUserDefinedToolRecommendedJobSetupTPV": 69.9,
|
||||
"integration.test_user_preferences::TestUserPreferences": 27.6,
|
||||
"integration.test_users::TestAdminResendActivationEmail": 29.8,
|
||||
"integration.test_users::TestExposeOnlyUserEmailIntegration": 28.9,
|
||||
"integration.test_users::TestExposeOnlyUserNameIntegration": 27.8,
|
||||
"integration.test_users::TestExposeUsersIntegration": 35.3,
|
||||
"integration.test_users::TestUnexposedUsersIntegration": 28.6,
|
||||
"integration.test_vault_extra_prefs::TestExtraUserPreferences": 33.9,
|
||||
"integration.test_vault_file_source::TestVaultFileSourceIntegration": 29.1,
|
||||
"integration.test_web_framework_config::TestAllowOriginIntegration": 35.9,
|
||||
"integration.test_web_framework_config::TestCorsDefaultIntegration": 28.3,
|
||||
"integration.test_work_queue_put_failure::TestWorkQueuePutFailure": 29.0,
|
||||
"integration.test_workflow_completion_hooks::TestWorkflowCompletionExportHook": 34.5,
|
||||
"integration.test_workflow_handler_configuration::TestDefaultWorkflowHandlerIfJobHandlerOff": 35.7,
|
||||
"integration.test_workflow_handler_configuration::TestDefaultWorkflowHandlerIfJobHandlerOn": 26.7,
|
||||
"integration.test_workflow_handler_configuration::TestDefaultWorkflowHandlerOn": 28.0,
|
||||
"integration.test_workflow_handler_configuration::TestExplicitWorkflowHandlersOff": 27.3,
|
||||
"integration.test_workflow_handler_configuration::TestExplicitWorkflowHandlersOffPool": 28.3,
|
||||
"integration.test_workflow_handler_configuration::TestExplicitWorkflowHandlersOn": 24.7,
|
||||
"integration.test_workflow_handler_configuration::TestHistoryParallelConfiguration": 26.4,
|
||||
"integration.test_workflow_handler_configuration::TestHistoryRestrictionConfiguration": 27.4,
|
||||
"integration.test_workflow_handler_configuration::TestJobHandlerAsWorkflowHandlerWithDbSkipLocked": 44.7,
|
||||
"integration.test_workflow_handler_configuration::TestJobHandlerAsWorkflowHandlerWithDbSkipLockedAttachToPool": 30.0,
|
||||
"integration.test_workflow_handler_configuration::TestWorkflowSchedulerHandlerAssignment": 39.0,
|
||||
"integration.test_workflow_handler_configuration::TestWorkflowSchedulerHandlerAssignmentDbSkipLocked": 28.6,
|
||||
"integration.test_workflow_handler_configuration::TestWorkflowSchedulerHandlerAssignmentDbTransactionIsolation": 28.4,
|
||||
"integration.test_workflow_invocation::TestWorkflowInvocation": 119.6,
|
||||
"integration.test_workflow_refactoring::TestWorkflowRefactoringIntegration": 51.2,
|
||||
"integration.test_workflow_scheduling_options::TestMaximumWorkflowInvocationDuration": 70.5,
|
||||
"integration.test_workflow_scheduling_options::TestMaximumWorkflowJobsPerSchedulingIteration": 55.1,
|
||||
"integration.test_workflow_sync::TestWorkflowSync": 34.1,
|
||||
"integration.test_workflow_tasks::TestWorkflowTasksIntegration": 398.0
|
||||
}
|
||||
@@ -7,10 +7,20 @@ from beaker.cache import CacheManager
|
||||
from beaker.util import parse_cache_config_options
|
||||
|
||||
from galaxy.tool_util.deps.mulled.util import NAMESPACE_HAS_REPO_NAME_KEY
|
||||
from galaxy_test.conftest import ( # noqa: F401
|
||||
pytest_configure,
|
||||
pytest_plugins,
|
||||
from galaxy_test import shard
|
||||
from galaxy_test.conftest import pytest_plugins # noqa: F401
|
||||
from galaxy_test.conftest import (
|
||||
pytest_configure as _base_pytest_configure,
|
||||
)
|
||||
from galaxy_test.shard import ( # noqa: F401
|
||||
pytest_collection_modifyitems,
|
||||
pytest_report_collectionfinish,
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
_base_pytest_configure(config)
|
||||
shard.pytest_configure(config)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
from collections import Counter
|
||||
|
||||
from galaxy_test.shard import (
|
||||
assign_groups_to_shards,
|
||||
build_groups,
|
||||
select_shard,
|
||||
ShardGroup,
|
||||
)
|
||||
|
||||
|
||||
class FakeModule:
|
||||
def __init__(self, name):
|
||||
self.__name__ = name
|
||||
|
||||
|
||||
class FakeItem:
|
||||
def __init__(self, module_name, class_name, node_id):
|
||||
self.module = FakeModule(module_name)
|
||||
self.cls = type(class_name, (), {}) if class_name else None
|
||||
self.nodeid = node_id
|
||||
|
||||
def __repr__(self):
|
||||
return self.nodeid
|
||||
|
||||
|
||||
def make_items(*specs):
|
||||
return [FakeItem(module, cls, f"{module}::{cls}::{name}") for module, cls, name in specs]
|
||||
|
||||
|
||||
def test_class_items_stay_in_one_group():
|
||||
items = make_items(
|
||||
("test_a", "TestOne", "test_x"),
|
||||
("test_a", "TestOne", "test_y"),
|
||||
("test_a", "TestTwo", "test_z"),
|
||||
)
|
||||
groups = {group.key: group for group in build_groups(items, {})}
|
||||
assert set(groups) == {"test_a::TestOne", "test_a::TestTwo"}
|
||||
assert len(groups["test_a::TestOne"].items) == 2
|
||||
|
||||
|
||||
def test_module_level_items_group_by_module():
|
||||
items = make_items(("test_a", None, "test_x"), ("test_a", None, "test_y"))
|
||||
groups = build_groups(items, {})
|
||||
assert len(groups) == 1
|
||||
assert groups[0].key == "test_a"
|
||||
|
||||
|
||||
def test_unknown_groups_get_a_default_weight():
|
||||
groups = build_groups(make_items(("test_a", "TestOne", "test_x")), {})
|
||||
assert groups[0].seconds > 0
|
||||
|
||||
|
||||
def test_recorded_durations_are_used():
|
||||
groups = build_groups(make_items(("test_a", "TestOne", "test_x")), {"test_a::TestOne": 123.0})
|
||||
assert groups[0].seconds == 123.0
|
||||
|
||||
|
||||
def test_longest_group_lands_on_its_own_shard():
|
||||
groups = [
|
||||
ShardGroup("slow", [], 100.0),
|
||||
ShardGroup("a", [], 10.0),
|
||||
ShardGroup("b", [], 10.0),
|
||||
ShardGroup("c", [], 10.0),
|
||||
]
|
||||
shards = assign_groups_to_shards(groups, 2)
|
||||
loads = sorted(sum(group.seconds for group in shard) for shard in shards)
|
||||
assert loads == [30.0, 100.0]
|
||||
|
||||
|
||||
def test_assignment_is_deterministic_for_equal_weights():
|
||||
groups = [ShardGroup(key, [], 10.0) for key in ("d", "a", "c", "b")]
|
||||
first = assign_groups_to_shards(groups, 3)
|
||||
second = assign_groups_to_shards(list(reversed(groups)), 3)
|
||||
assert [[g.key for g in shard] for shard in first] == [[g.key for g in shard] for shard in second]
|
||||
|
||||
|
||||
def test_shards_partition_the_suite_without_splitting_classes():
|
||||
items = make_items(
|
||||
*[
|
||||
(f"test_mod{module}", f"TestClass{cls}", f"test_{index}")
|
||||
for module in range(5)
|
||||
for cls in range(3)
|
||||
for index in range(4)
|
||||
]
|
||||
)
|
||||
num_shards = 4
|
||||
selected = [select_shard(items, shard_id, num_shards, {}) for shard_id in range(num_shards)]
|
||||
|
||||
node_ids = [item.nodeid for shard in selected for item in shard]
|
||||
assert Counter(node_ids) == Counter(item.nodeid for item in items), "shards must partition the suite exactly"
|
||||
|
||||
shards_per_group: Counter[str] = Counter()
|
||||
for shard in selected:
|
||||
for key in {f"{item.module.__name__}::{item.cls.__name__}" for item in shard}:
|
||||
shards_per_group[key] += 1
|
||||
assert all(count == 1 for count in shards_per_group.values()), "a test class must not span shards"
|
||||
|
||||
|
||||
def test_single_shard_selects_everything():
|
||||
items = make_items(("test_a", "TestOne", "test_x"), ("test_b", "TestTwo", "test_y"))
|
||||
assert select_shard(items, 0, 1, {}) == items
|
||||
Reference in New Issue
Block a user