diff --git a/lib/galaxy_test/shard.py b/lib/galaxy_test/shard.py new file mode 100644 index 00000000000..140cc3c1605 --- /dev/null +++ b/lib/galaxy_test/shard.py @@ -0,0 +1,121 @@ +"""Distribute tests across CI shards without splitting up test classes. + +An integration test class boots a Galaxy instance in ``setUpClass``, so a class split +across shards makes every one of them pay that startup. Whole classes are kept together +and packed into shards longest-first, weighted by recorded durations, which balances +shards by cost rather than by name. +""" + +import json +import os +from collections import defaultdict +from typing import ( + Any, + NamedTuple, +) + +import pytest + +DURATIONS_ENV_VAR = "GALAXY_TEST_SHARD_DURATIONS" +# Per-group seconds measured from a CI run, as {"module::Class": seconds}. Only affects +# balance, never correctness: unknown groups get DEFAULT_GROUP_SECONDS. Regenerate from a +# full unsharded run with --structured_data_report_file, summing each test's setup, call +# and teardown into its group_key(). GALAXY_TEST_SHARD_DURATIONS overrides the path. +DEFAULT_DURATIONS_FILE = os.path.join(os.path.dirname(__file__), "shard_durations.json") +# What to assume a group costs when we have no measurement for it. Integration classes +# are startup-dominated, so an unknown group is much closer to one startup than to zero. +DEFAULT_GROUP_SECONDS = 40.0 + + +class ShardGroup(NamedTuple): + """Tests that share a Galaxy instance, and therefore must share a shard.""" + + key: str + items: list[Any] + seconds: float + + +def group_key(item) -> str: + """Identify the instance an item belongs to. + + Class for class-based tests, module for those taking a module-scoped instance + fixture, and the file for items with no module at all. + """ + module = getattr(item, "module", None) + if module is None: + return item.nodeid.split("::")[0] + cls = getattr(item, "cls", None) + if cls is not None: + return f"{module.__name__}::{cls.__name__}" + return module.__name__ + + +def load_durations(path: str | None = None) -> dict[str, float]: + """Load recorded per-group durations, falling back to an empty mapping.""" + path = path or os.environ.get(DURATIONS_ENV_VAR) or DEFAULT_DURATIONS_FILE + try: + with open(path) as fh: + return {key: float(seconds) for key, seconds in json.load(fh).items()} + except (OSError, ValueError): + return {} + + +def build_groups(items: list[Any], durations: dict[str, float]) -> list[ShardGroup]: + grouped: dict[str, list[Any]] = defaultdict(list) + for item in items: + grouped[group_key(item)].append(item) + return [ + ShardGroup(key, group_items, durations.get(key, DEFAULT_GROUP_SECONDS)) for key, group_items in grouped.items() + ] + + +def assign_groups_to_shards(groups: list[ShardGroup], num_shards: int) -> list[list[ShardGroup]]: + """Pack groups into shards longest-first, always onto the shard with the least work. + + Greedy bin-packing. Ties break on group key: every shard runs this independently and + the assignments must agree, or tests get dropped or duplicated. + """ + shards: list[list[ShardGroup]] = [[] for _ in range(num_shards)] + loads = [0.0] * num_shards + for group in sorted(groups, key=lambda g: (-g.seconds, g.key)): + target = min(range(num_shards), key=lambda index: (loads[index], index)) + shards[target].append(group) + loads[target] += group.seconds + return shards + + +def select_shard(items: list[Any], shard_id: int, num_shards: int, durations: dict[str, float]) -> list[Any]: + groups = build_groups(items, durations) + assigned = assign_groups_to_shards(groups, num_shards)[shard_id] + selected = [] + for group in sorted(assigned, key=lambda g: g.key): + selected.extend(group.items) + return selected + + +def pytest_configure(config) -> None: + """Take over sharding from pytest-shard, reusing its command line options.""" + shard_plugin = config.pluginmanager.get_plugin("pytest-shard") + if shard_plugin is not None: + config.pluginmanager.unregister(shard_plugin) + + +def pytest_report_collectionfinish(config, items) -> str | None: + """Report which groups this shard runs.""" + num_shards = config.getoption("num_shards", default=1) + if num_shards <= 1: + return None + groups = sorted({group_key(item) for item in items}) + shard_id = config.getoption("shard_id", default=0) + return f"Running {len(items)} items from {len(groups)} groups in shard {shard_id}: {', '.join(groups)}" + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items: list[Any]) -> None: + num_shards = config.getoption("num_shards", default=1) + shard_id = config.getoption("shard_id", default=0) + if num_shards <= 1: + return + if shard_id >= num_shards: + raise ValueError(f"shard_id = {shard_id} must be less than num_shards = {num_shards}") + items[:] = select_shard(items, shard_id, num_shards, load_durations()) diff --git a/lib/galaxy_test/shard_durations.json b/lib/galaxy_test/shard_durations.json new file mode 100644 index 00000000000..9472c4d9d60 --- /dev/null +++ b/lib/galaxy_test/shard_durations.json @@ -0,0 +1,271 @@ +{ + "integration.objectstore.test_azure": 10.9, + "integration.objectstore.test_bulk_storage_operations::TestBulkStorageOperationsIntegration": 499.4, + "integration.objectstore.test_changing_objectstore::TestChangingStoreObjectStoreIntegration": 42.7, + "integration.objectstore.test_direct_download_redirect::TestDirectDownloadRedirectIntegration": 47.4, + "integration.objectstore.test_jobs": 38.1, + "integration.objectstore.test_jobs::TestObjectStoreJobsIntegration": 37.2, + "integration.objectstore.test_mixed_store_by::TestMixedStoreByObjectStoreIntegration": 35.1, + "integration.objectstore.test_objectstore_datatype_upload": 814.5, + "integration.objectstore.test_onedata_objectstore": 447.7, + "integration.objectstore.test_per_user::TestPerUserObjectStoreIntegration": 64.7, + "integration.objectstore.test_per_user::TestPerUserObjectStoreQuotaIntegration": 52.0, + "integration.objectstore.test_per_user::TestPerUserObjectStoreUpgradesIntegration": 31.0, + "integration.objectstore.test_per_user::TestPerUserObjectStoreUpgradesWithSecretsIntegration": 72.0, + "integration.objectstore.test_per_user::TestPerUserObjectStoreWithExtendedMetadataIntegration": 45.1, + "integration.objectstore.test_per_user::TestPerUserObjectStoreWithSecretsIntegration": 68.5, + "integration.objectstore.test_private_handling::TestPrivateCannotWritePublicDataObjectStoreIntegration": 30.1, + "integration.objectstore.test_private_handling::TestPrivatePreventsSharingObjectStoreIntegration": 41.4, + "integration.objectstore.test_private_handling_library_imports::TestToolOutputFromLibraryImportWithPrivateObjectStore": 42.9, + "integration.objectstore.test_quota_limit::TestQuotaIntegration": 51.8, + "integration.objectstore.test_remote_objectstore_cache_operations::TestCacheOperation": 39.4, + "integration.objectstore.test_remote_objectstore_cache_operations::TestCacheOperationWithNoCacheUpdate": 34.7, + "integration.objectstore.test_rucio_objectstore": 343.0, + "integration.objectstore.test_selection_with_resource_parameters::TestObjectStoreSelectionWithResourceParameterIntegration": 34.9, + "integration.objectstore.test_selection_with_user_preferred_object_store::TestObjectStoreSelectionWithExtendedMetadataIntegration": 322.9, + "integration.objectstore.test_selection_with_user_preferred_object_store::TestObjectStoreSelectionWithPreferredObjectStoresIntegration": 130.3, + "integration.objectstore.test_swift_objectstore": 212.9, + "integration.oidc.test_auth_oidc::TestFixedDelegatedAuthIntegration": 77.1, + "integration.oidc.test_auth_oidc::TestGalaxyOIDCLoginIntegration": 107.9, + "integration.oidc.test_auth_oidc::TestWithoutFixedDelegatedAuth": 66.8, + "integration.test_agents::TestAgentOperationsManagerEncoding": 29.3, + "integration.test_agents::TestAgentsApi": 32.6, + "integration.test_agents::TestAgentsApiLiveLLM": 0.1, + "integration.test_agents::TestMCPServerSmoke": 66.7, + "integration.test_apache_nginx_sendfile::TestApacheSendFileHeader": 46.1, + "integration.test_apache_nginx_sendfile::TestNginxAccelHeader": 42.8, + "integration.test_async_downloads::TestAsyncDownloadsIntegration": 29.7, + "integration.test_celery_fetch_quota::TestCeleryFetchQuotaEnforcement": 36.9, + "integration.test_celery_tasks::TestCeleryTasksIntegration": 42.9, + "integration.test_celery_user_concurrency_limit::TestCeleryUserConcurrencyLimitDisabled": 30.5, + "integration.test_celery_user_concurrency_limit::TestCeleryUserConcurrencyLimitPostgres": 47.9, + "integration.test_celery_user_concurrency_limit::TestCeleryUserConcurrencyLimitSqlite": 37.6, + "integration.test_celery_user_rate_limit::TestCeleryUserRateLimitIntegrationNoLimit": 27.9, + "integration.test_celery_user_rate_limit::TestCeleryUserRateLimitIntegrationPostgres": 48.5, + "integration.test_celery_user_rate_limit::TestCeleryUserRateLimitIntegrationSqlite": 51.6, + "integration.test_chained_dynamic_destinations::TestChainedDynamicDestinationIntegration": 34.9, + "integration.test_cli_runners::TestParamikoCliOpenPBSIntegration": 43.8, + "integration.test_cli_runners::TestParamikoCliSlurmIntegration": 66.4, + "integration.test_cli_runners::TestShellJobCliOpenPBSIntegration": 61.2, + "integration.test_cli_runners::TestShellJobCliSlurmIntegration": 80.1, + "integration.test_coexecution::TestCoexecution": 0.1, + "integration.test_coexecution::TestKubernetesDependencyResolutionIntegration": 0.1, + "integration.test_coexecution::TestKubernetesStagingContainerIntegration": 0.1, + "integration.test_coexecution::TestTesCoexecutionContainerIntegration": 0.1, + "integration.test_coexecution::TestTesCoexecutionCustomAmqpKeyContainerIntegration": 0.1, + "integration.test_coexecution::TestTesDependencyResolutionIntegration": 0.1, + "integration.test_config_options_users::TestDefaultUserExposeIntegration": 27.8, + "integration.test_config_options_users::TestEmailUserExposeIntegration": 34.3, + "integration.test_config_options_users::TestUsernameUserExposeIntegration": 32.5, + "integration.test_config_schema::TestConfigSchema": 31.4, + "integration.test_configuration_decode::TestConfigurationDecodeIntegration": 33.2, + "integration.test_container_resolvers::TestCachedExplicitSingularityContainerResolver": 45.0, + "integration.test_container_resolvers::TestCachedExplicitSingularityContainerResolverWithNamespace": 48.6, + "integration.test_container_resolvers::TestCachedExplicitSingularityContainerResolverWithSingularityRequirement": 39.8, + "integration.test_container_resolvers::TestCondaFallBack": 49.5, + "integration.test_container_resolvers::TestCondaFallBackAndRequireContainer": 46.1, + "integration.test_container_resolvers::TestDefaultContainerResolvers": 73.4, + "integration.test_container_resolvers::TestDefaultSingularityContainerResolvers": 65.4, + "integration.test_container_resolvers::TestExplicitContainerResolver": 34.1, + "integration.test_container_resolvers::TestExplicitSingularityContainerResolver": 38.1, + "integration.test_container_resolvers::TestMulledContainerResolvers": 46.2, + "integration.test_container_resolvers::TestMulledContainerResolversNoAutoInstall": 42.3, + "integration.test_container_resolvers::TestMulledSingularityContainerResolvers": 37.8, + "integration.test_container_resolvers::TestMulledSingularityContainersResolversNoAutoInstall": 36.3, + "integration.test_containerized_jobs::TestDockerizedJobsIntegration": 146.3, + "integration.test_containerized_jobs::TestInlineContainerConfiguration": 30.1, + "integration.test_containerized_jobs::TestInlineJobEnvironmentContainerResolver": 36.4, + "integration.test_containerized_jobs::TestMappingContainerResolver": 37.9, + "integration.test_containerized_jobs::TestPerDestinationContainerConfiguration": 36.7, + "integration.test_containerized_jobs::TestSingularityJobsIntegration": 174.1, + "integration.test_credentials::TestCredentialsApi": 39.9, + "integration.test_data_manager::TestDataManagerIntegration": 73.9, + "integration.test_data_manager_refgenie::TestDataManagerIntegration": 45.5, + "integration.test_data_manager_workflow_bundle::TestDataManagerWorkflowInvocation": 68.9, + "integration.test_dataset_copy_metadata_files::TestDirectoryStrategyMetadataFileIntegrationTestCase": 52.4, + "integration.test_dataset_copy_metadata_files::TestExtendedMetadataStrategyMetadataFileIntegrationTestCase": 51.2, + "integration.test_dataset_hashing::TestDatasetHashingAlwaysIntegration": 30.7, + "integration.test_dataset_hashing::TestDatasetHashingIntegration": 38.6, + "integration.test_dataset_hashing::TestDatasetHashingNeverIntegration": 34.3, + "integration.test_dataset_hashing::TestDatasetHashingUploadIntegration": 35.2, + "integration.test_datatype_upload": 589.0, + "integration.test_default_permissions::TestDefaultPermissionsIntegration": 32.2, + "integration.test_default_permissions::TestPrivateDefaultPermissionsIntegration": 28.8, + "integration.test_default_permissions::TestPublicDefaultPermissionsIntegration": 37.7, + "integration.test_drs_compact_identifiers::TestDRSCompactIdentifiersIntegration": 0.1, + "integration.test_dynamic_edam_loading::TestDynamicEdamLoadingIntegration": 34.4, + "integration.test_edam_toolbox::TestEdamToolboxDefaultIntegration": 29.5, + "integration.test_edam_toolbox::TestEdamToolboxIntegration": 28.3, + "integration.test_entry_point_sse::TestEntryPointSSEIntegration": 60.8, + "integration.test_error_report::TestErrorEmailReportIntegration": 32.1, + "integration.test_error_report::TestErrorReportIntegration": 35.4, + "integration.test_event_loop_blocking::TestAiocopBlockingDetection": 31.0, + "integration.test_extended_metadata": 895.6, + "integration.test_extended_metadata::TestExtendedMetadataDeferredIntegration": 48.7, + "integration.test_extended_metadata::TestExtendedMetadataIntegration": 169.7, + "integration.test_extended_metadata_mapping::TestExtendedMetadataMappingIntegration": 56.0, + "integration.test_extended_metadata_outputs_to_working_directory": 509.5, + "integration.test_fail_job_tool_unavailable::TestFailJobWhenToolUnavailable": 49.3, + "integration.test_flush_per_n_datasets": 41.5, + "integration.test_galaxy_interactor::TestGalaxyInteractor": 29.8, + "integration.test_genomes::TestGenomes": 31.0, + "integration.test_handler_assignment_methods::TestDBPreassignHandlerAssignmentMethodIntegration": 36.6, + "integration.test_handler_assignment_methods::TestDBSkipLockedHandlerAssignmentMethodIntegration": 28.6, + "integration.test_handler_assignment_methods::TestDBTransactionIsolationHandlerAssignmentMethodIntegration": 38.4, + "integration.test_hashicorp_vault::TestHashicorpVaultRenewalGalaxyIntegration": 38.2, + "integration.test_history_archiving::TestHistoryArchivingAdmin": 75.4, + "integration.test_history_archiving::TestHistoryArchivingWithExportRecord": 65.1, + "integration.test_history_import_export::TestImportExportHistoryContentsViaTasksIntegration": 40.5, + "integration.test_history_import_export::TestImportExportHistoryOutputsToWorkingDirIntegration": 215.6, + "integration.test_history_import_export::TestImportExportHistoryViaTasksIntegration": 93.3, + "integration.test_history_sse::TestHistorySSEIntegration": 44.9, + "integration.test_htcondor_runner": 193.0, + "integration.test_htcondor_runner::TestHTCondorContainerJob": 69.4, + "integration.test_interactivetools_api::TestInteractiveToolsIntegration": 82.1, + "integration.test_interactivetools_api::TestInteractiveToolsPulsarIntegration": 61.1, + "integration.test_interactivetools_api::TestInteractiveToolsRemoteProxyIntegration": 0.9, + "integration.test_interactivetools_api::TestInteractiveToolsShortURLIntegration": 59.0, + "integration.test_interactivetools_api::TestKubeInteractiveToolsRemoteProxyIntegration": 0.1, + "integration.test_job_cache::TestJobCacheFiltering": 32.9, + "integration.test_job_environments::TestDefaultJobEnvironmentIntegration": 40.2, + "integration.test_job_environments::TestEmbeddedPulsarDefaultJobEnvironmentIntegration": 44.5, + "integration.test_job_environments::TestJobIOEnvironmentIntegration": 29.3, + "integration.test_job_environments::TestSharedHomeJobEnvironmentIntegration": 41.0, + "integration.test_job_environments::TestTmpDirAsShellCommandJobEnvironmentIntegration": 31.4, + "integration.test_job_environments::TestTmpDirToTrueJobEnvironmentIntegration": 36.2, + "integration.test_job_files::TestJobFilesIntegration": 33.1, + "integration.test_job_files_tus": 48.9, + "integration.test_job_outputs_to_working_directory": 39.0, + "integration.test_job_recovery::TestJobRecoveryAfterHandledIntegration": 44.7, + "integration.test_job_recovery::TestJobRecoveryBeforeHandledIntegration": 37.5, + "integration.test_job_resource_error_recovery::TestJobRecoveryBeforeHandledIntegration": 29.4, + "integration.test_job_resubmission::TestJobResubmissionDefaultIntegration": 39.3, + "integration.test_job_resubmission::TestJobResubmissionDynamicIntegration": 29.5, + "integration.test_job_resubmission::TestJobResubmissionDynamicMultipleIntegration": 28.3, + "integration.test_job_resubmission::TestJobResubmissionIntegration": 138.1, + "integration.test_job_resubmission::TestJobResubmissionPulsarIntegration": 39.3, + "integration.test_job_resubmission::TestJobResubmissionSmallMemoryIntegration": 29.1, + "integration.test_job_resubmission::TestJobResubmissionSmallMemoryResubmitsToLargeIntegration": 29.5, + "integration.test_job_resubmission::TestJobResubmissionToolDetectedErrorIntegration": 38.0, + "integration.test_job_resubmission::TestJobResubmissionToolDetectedErrorResubmitsIntegration": 30.1, + "integration.test_kubernetes_runner::TestKubernetesIntegration": 185.3, + "integration.test_landing_requests::TestLandingRequestsIntegration": 31.5, + "integration.test_landing_requests::TestLandingRequestsWithoutHeadersConfigIntegration": 37.5, + "integration.test_landing_requests::TestLandingRequestsWithoutVaultIntegration": 35.4, + "integration.test_legacy_store_by": 47.4, + "integration.test_legacy_store_by::TestChangeDatatypeStoreByIdIntegration": 46.1, + "integration.test_live_evals::TestLiveEvals": 0.1, + "integration.test_local_job_cancellation::TestLocalJobCancellation": 42.0, + "integration.test_materialize_dataset_instance_tasks::TestMaterializeDatasetInstanceTasaksIntegration": 57.2, + "integration.test_max_discovered_files::TestExtendedMetadataMaxDiscoveredFiles": 39.5, + "integration.test_max_discovered_files::TestMaxDiscoveredFiles": 29.9, + "integration.test_metadata_strategy::TestDiskUsageCeleryExtended": 40.8, + "integration.test_metadata_strategy::TestDiskUsageUpdateDefault": 32.5, + "integration.test_model_store_scripts::TestModelStoreScriptsIntegration": 30.7, + "integration.test_notification_sse::TestNotificationSSEIntegration": 52.6, + "integration.test_notifications::TestNotificationsIntegration": 39.0, + "integration.test_notifications::TestNotificationsIntegrationTaskBased": 41.7, + "integration.test_page_revision_json_encoding::TestPageJsonEncodingIntegration": 33.4, + "integration.test_panel_views::TestPanelViewsFromConfigIntegration": 28.5, + "integration.test_panel_views::TestPanelViewsFromDirectoryIntegration": 32.9, + "integration.test_panel_views::TestPanelViewsWithShedTools": 41.1, + "integration.test_plugins::TestPluginsInferenceServicesConfig": 26.2, + "integration.test_plugins::TestPluginsInferenceServicesDefault": 26.5, + "integration.test_plugins::TestVisualizationPluginsApi": 36.5, + "integration.test_prefix_handling::TestPrefixUrlSerializationIntegration": 35.8, + "integration.test_pulsar_embedded": 370.5, + "integration.test_pulsar_embedded::TestEmbeddedPulsarIntegrationInstance": 70.0, + "integration.test_pulsar_embedded_containers": 33.5, + "integration.test_pulsar_embedded_containers::TestEmbeddedDockerPulsarIntegration": 35.8, + "integration.test_pulsar_embedded_containers::TestEmbeddedSingularityPulsarIntegration": 44.2, + "integration.test_pulsar_embedded_copy_working": 42.4, + "integration.test_pulsar_embedded_extended_metadata": 95.1, + "integration.test_pulsar_embedded_mq": 84.0, + "integration.test_pulsar_embedded_mq::TestEmbeddedMessageQueuePulsarExtendedMetadataPurge": 42.5, + "integration.test_pulsar_embedded_mq::TestEmbeddedMessageQueuePulsarPurge": 47.6, + "integration.test_pulsar_embedded_none": 48.9, + "integration.test_pulsar_embedded_relay": 74.4, + "integration.test_pulsar_embedded_remote_metadata": 67.7, + "integration.test_purge_datasets::TestPurgeDatasetsIntegration": 60.5, + "integration.test_purge_datasets::TestPurgeDatasetsWithoutCeleryIntegration": 102.7, + "integration.test_quota::TestQuotaIntegration": 34.7, + "integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageHierarchicalIntegration": 31.3, + "integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageHierarchicalNoTaskIntegration": 50.0, + "integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageIntegration": 33.8, + "integration.test_recalculate_user_disk_usage::TestRecalculateUserDiskUsageSimpleHierarchicalIntegration": 31.2, + "integration.test_remote_files::TestRemoteFilesIntegration": 194.9, + "integration.test_remote_files::TestRemoteFilesNotConfiguredIntegration": 29.1, + "integration.test_remote_files_histories::TestRemoteFilesHistoryImportExportIntegration": 61.5, + "integration.test_remote_files_posix::TestPosixFileSourceIntegration": 33.6, + "integration.test_remote_files_posix::TestPreferLinksPosixFileSourceIntegration": 61.7, + "integration.test_repository_operations::TestRepositoryInstallIntegrationTestCase": 71.1, + "integration.test_resolvers::TestCondaResolutionIntegration": 115.4, + "integration.test_save_job_id_on_datasets": 102.1, + "integration.test_scripts::TestScriptsIntegration": 145.3, + "integration.test_scripts_pgcleanup::TestPgCleanupUserObjectStoreIntegration": 50.7, + "integration.test_scripts_pgcleanup::TestScriptsPgCleanupIntegration": 169.2, + "integration.test_shed_tool_tests::TestToolShedToolTestIntegration": 59.1, + "integration.test_storage_cleaner::TestStorageCleaner": 40.1, + "integration.test_structured_dataset::TestStructuredDataset": 46.3, + "integration.test_structured_like_unpopulated::TestStructuredLikeUnpopulatedRaisesNotReady": 29.4, + "integration.test_tool_data_bundles::TestDataBundlesIntegration": 62.4, + "integration.test_tool_data_delete::TestAdminToolDataIntegration": 47.7, + "integration.test_tool_submission_errors::TestFailJobWhenToolUnavailable": 38.8, + "integration.test_upload_configuration_options::TestAdminsCanPasteFilePaths": 47.4, + "integration.test_upload_configuration_options::TestAdvancedFtpUploadFetch": 29.5, + "integration.test_upload_configuration_options::TestAutoDecompress": 35.3, + "integration.test_upload_configuration_options::TestDefaultBinaryContentFilters": 29.6, + "integration.test_upload_configuration_options::TestDirectoryAndCompressedTypes": 33.4, + "integration.test_upload_configuration_options::TestDisableContentChecking": 30.4, + "integration.test_upload_configuration_options::TestDisableFtpPurgeUploadConfiguration": 30.4, + "integration.test_upload_configuration_options::TestEnableFtpPurgeUploadConfiguration": 30.4, + "integration.test_upload_configuration_options::TestExplicitEmailAsIdentifierFtpUploadConfiguration": 40.7, + "integration.test_upload_configuration_options::TestFetchByPath": 47.8, + "integration.test_upload_configuration_options::TestInvalidFetchRequests": 29.5, + "integration.test_upload_configuration_options::TestLinkDataUploadExtendedMetadata": 34.2, + "integration.test_upload_configuration_options::TestLocalAddressWhitelisting": 29.2, + "integration.test_upload_configuration_options::TestNonAdminsCannotPasteFilePath": 30.4, + "integration.test_upload_configuration_options::TestPerUsernameFtpUploadConfiguration": 35.1, + "integration.test_upload_configuration_options::TestServerDirectoryOffByDefault": 26.4, + "integration.test_upload_configuration_options::TestServerDirectoryValidUsage": 40.2, + "integration.test_upload_configuration_options::TestSimpleFtpUploadConfiguration": 29.3, + "integration.test_upload_configuration_options::TestTemplatedFtpDirectoryUploadConfiguration": 30.2, + "integration.test_upload_configuration_options::TestUploadOptionsFtpUploadConfiguration": 52.2, + "integration.test_upload_configuration_options::TestUploadWithDirectoryMetadata": 30.6, + "integration.test_upload_configuration_options::TestUploadWithExtendedMetadata": 29.3, + "integration.test_upload_configuration_options::TestUserServerDirectoryOffByDefault": 26.2, + "integration.test_upload_configuration_options::TestUserServerDirectoryValidUsage": 42.5, + "integration.test_user_defined_tool_job_conf::TestUserDefinedToolRecommendedJobSetup": 49.4, + "integration.test_user_defined_tool_job_conf::TestUserDefinedToolRecommendedJobSetupTPV": 69.9, + "integration.test_user_preferences::TestUserPreferences": 27.6, + "integration.test_users::TestAdminResendActivationEmail": 29.8, + "integration.test_users::TestExposeOnlyUserEmailIntegration": 28.9, + "integration.test_users::TestExposeOnlyUserNameIntegration": 27.8, + "integration.test_users::TestExposeUsersIntegration": 35.3, + "integration.test_users::TestUnexposedUsersIntegration": 28.6, + "integration.test_vault_extra_prefs::TestExtraUserPreferences": 33.9, + "integration.test_vault_file_source::TestVaultFileSourceIntegration": 29.1, + "integration.test_web_framework_config::TestAllowOriginIntegration": 35.9, + "integration.test_web_framework_config::TestCorsDefaultIntegration": 28.3, + "integration.test_work_queue_put_failure::TestWorkQueuePutFailure": 29.0, + "integration.test_workflow_completion_hooks::TestWorkflowCompletionExportHook": 34.5, + "integration.test_workflow_handler_configuration::TestDefaultWorkflowHandlerIfJobHandlerOff": 35.7, + "integration.test_workflow_handler_configuration::TestDefaultWorkflowHandlerIfJobHandlerOn": 26.7, + "integration.test_workflow_handler_configuration::TestDefaultWorkflowHandlerOn": 28.0, + "integration.test_workflow_handler_configuration::TestExplicitWorkflowHandlersOff": 27.3, + "integration.test_workflow_handler_configuration::TestExplicitWorkflowHandlersOffPool": 28.3, + "integration.test_workflow_handler_configuration::TestExplicitWorkflowHandlersOn": 24.7, + "integration.test_workflow_handler_configuration::TestHistoryParallelConfiguration": 26.4, + "integration.test_workflow_handler_configuration::TestHistoryRestrictionConfiguration": 27.4, + "integration.test_workflow_handler_configuration::TestJobHandlerAsWorkflowHandlerWithDbSkipLocked": 44.7, + "integration.test_workflow_handler_configuration::TestJobHandlerAsWorkflowHandlerWithDbSkipLockedAttachToPool": 30.0, + "integration.test_workflow_handler_configuration::TestWorkflowSchedulerHandlerAssignment": 39.0, + "integration.test_workflow_handler_configuration::TestWorkflowSchedulerHandlerAssignmentDbSkipLocked": 28.6, + "integration.test_workflow_handler_configuration::TestWorkflowSchedulerHandlerAssignmentDbTransactionIsolation": 28.4, + "integration.test_workflow_invocation::TestWorkflowInvocation": 119.6, + "integration.test_workflow_refactoring::TestWorkflowRefactoringIntegration": 51.2, + "integration.test_workflow_scheduling_options::TestMaximumWorkflowInvocationDuration": 70.5, + "integration.test_workflow_scheduling_options::TestMaximumWorkflowJobsPerSchedulingIteration": 55.1, + "integration.test_workflow_sync::TestWorkflowSync": 34.1, + "integration.test_workflow_tasks::TestWorkflowTasksIntegration": 398.0 +} diff --git a/test/integration/conftest.py b/test/integration/conftest.py index a2d8b6b5a49..bb9d878b9b6 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -7,10 +7,20 @@ from beaker.cache import CacheManager from beaker.util import parse_cache_config_options from galaxy.tool_util.deps.mulled.util import NAMESPACE_HAS_REPO_NAME_KEY -from galaxy_test.conftest import ( # noqa: F401 - pytest_configure, - pytest_plugins, +from galaxy_test import shard +from galaxy_test.conftest import pytest_plugins # noqa: F401 +from galaxy_test.conftest import ( + pytest_configure as _base_pytest_configure, ) +from galaxy_test.shard import ( # noqa: F401 + pytest_collection_modifyitems, + pytest_report_collectionfinish, +) + + +def pytest_configure(config): + _base_pytest_configure(config) + shard.pytest_configure(config) @pytest.fixture(scope="session", autouse=True) diff --git a/test/unit/test_shard.py b/test/unit/test_shard.py new file mode 100644 index 00000000000..b68946fe663 --- /dev/null +++ b/test/unit/test_shard.py @@ -0,0 +1,101 @@ +from collections import Counter + +from galaxy_test.shard import ( + assign_groups_to_shards, + build_groups, + select_shard, + ShardGroup, +) + + +class FakeModule: + def __init__(self, name): + self.__name__ = name + + +class FakeItem: + def __init__(self, module_name, class_name, node_id): + self.module = FakeModule(module_name) + self.cls = type(class_name, (), {}) if class_name else None + self.nodeid = node_id + + def __repr__(self): + return self.nodeid + + +def make_items(*specs): + return [FakeItem(module, cls, f"{module}::{cls}::{name}") for module, cls, name in specs] + + +def test_class_items_stay_in_one_group(): + items = make_items( + ("test_a", "TestOne", "test_x"), + ("test_a", "TestOne", "test_y"), + ("test_a", "TestTwo", "test_z"), + ) + groups = {group.key: group for group in build_groups(items, {})} + assert set(groups) == {"test_a::TestOne", "test_a::TestTwo"} + assert len(groups["test_a::TestOne"].items) == 2 + + +def test_module_level_items_group_by_module(): + items = make_items(("test_a", None, "test_x"), ("test_a", None, "test_y")) + groups = build_groups(items, {}) + assert len(groups) == 1 + assert groups[0].key == "test_a" + + +def test_unknown_groups_get_a_default_weight(): + groups = build_groups(make_items(("test_a", "TestOne", "test_x")), {}) + assert groups[0].seconds > 0 + + +def test_recorded_durations_are_used(): + groups = build_groups(make_items(("test_a", "TestOne", "test_x")), {"test_a::TestOne": 123.0}) + assert groups[0].seconds == 123.0 + + +def test_longest_group_lands_on_its_own_shard(): + groups = [ + ShardGroup("slow", [], 100.0), + ShardGroup("a", [], 10.0), + ShardGroup("b", [], 10.0), + ShardGroup("c", [], 10.0), + ] + shards = assign_groups_to_shards(groups, 2) + loads = sorted(sum(group.seconds for group in shard) for shard in shards) + assert loads == [30.0, 100.0] + + +def test_assignment_is_deterministic_for_equal_weights(): + groups = [ShardGroup(key, [], 10.0) for key in ("d", "a", "c", "b")] + first = assign_groups_to_shards(groups, 3) + second = assign_groups_to_shards(list(reversed(groups)), 3) + assert [[g.key for g in shard] for shard in first] == [[g.key for g in shard] for shard in second] + + +def test_shards_partition_the_suite_without_splitting_classes(): + items = make_items( + *[ + (f"test_mod{module}", f"TestClass{cls}", f"test_{index}") + for module in range(5) + for cls in range(3) + for index in range(4) + ] + ) + num_shards = 4 + selected = [select_shard(items, shard_id, num_shards, {}) for shard_id in range(num_shards)] + + node_ids = [item.nodeid for shard in selected for item in shard] + assert Counter(node_ids) == Counter(item.nodeid for item in items), "shards must partition the suite exactly" + + shards_per_group: Counter[str] = Counter() + for shard in selected: + for key in {f"{item.module.__name__}::{item.cls.__name__}" for item in shard}: + shards_per_group[key] += 1 + assert all(count == 1 for count in shards_per_group.values()), "a test class must not span shards" + + +def test_single_shard_selects_everything(): + items = make_items(("test_a", "TestOne", "test_x"), ("test_b", "TestTwo", "test_y")) + assert select_shard(items, 0, 1, {}) == items