mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-31 18:42:33 +08:00
paperbenni
bec3d3ad9c
Drop Python 3.8 support in 5 Pulsar-compatible packages (job_metrics, tool_util, tool_util_models, util, objectstore) and run pyupgrade --py310-plus on their source code. - Bump requires-python from >=3.8 to >=3.10 in all 5 packages - Remove Python 3.8 and 3.9 classifiers - Remove ruff per-file-ignores for UP rules on these paths - Remove backports.zoneinfo conditional dependency - Remove pydyf<0.11 pin from conditional-requirements.txt - Update Makefile pyupgrade target (remove PY38_PYUPGRADE_PATHS) - Update CI workflow to test with Python 3.10 instead of 3.8 Clean up unused deprecated typing imports after pyupgrade Remove now-unused typing imports (Dict, List, Optional, Set, Tuple, Type, Union) that became dead after pyupgrade --py310-plus converted annotations to use built-in types and | syntax. Also run ruff check --fix --select=UP007,UP045 across the entire codebase to convert remaining Optional[X] -> X | None and Union[X, Y] -> X | Y patterns. Enable ruff UP007/UP045 for Python 3.10 union syntax Remove UP007 (Union[X,Y] -> X | Y) and UP045 (Optional[X] -> X | None) from the ruff ignore list and convert all type aliases across the codebase. These rules were deferred while Python 3.9 was supported; requires-python is now >=3.10. A custom script was used because neither ruff --fix nor pyupgrade --py310-plus converts Optional[X]/Union[X,Y] in type alias positions (e.g. X = Union[A, B]) — they only handle annotation positions (e.g. def f(x: Optional[int])). All 167 violations were module-level type aliases. A few edge cases were fixed manually: single-element Union[X,], typing.Union qualified refs, runtime Optional[type] calls, and Annotated[Optional[...]] pydantic fields. Fix UP007 autofix regression with string forward reference type aliases Commit fa6bd955a0 enabled ruff UP007/UP045 and auto-fixed module-level type aliases using Union with string forward references, producing invalid 'str | str' expressions. This was a known ruff bug (charliermarsh/ruff#826) that has since been fixed in later ruff versions, but this codebase was converted before the fix was in place. Revert to Union syntax and restore TYPE_CHECKING imports that ruff's TCH rule cleaned up as a side effect when it thought the forward references were unused.
90 lines
3.2 KiB
Python
90 lines
3.2 KiB
Python
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from galaxy.files import ConfiguredFileSources
|
|
|
|
|
|
def get_file_sources(file_sources_path):
|
|
assert os.path.exists(file_sources_path), f"file sources path [{file_sources_path}] does not exist"
|
|
with open(file_sources_path) as f:
|
|
file_sources_as_dict = json.load(f)
|
|
file_sources = ConfiguredFileSources.from_dict(file_sources_as_dict)
|
|
return file_sources
|
|
|
|
|
|
def check_for_duplicate_name(files_to_export):
|
|
seen = set()
|
|
duplicates = set()
|
|
for entry in files_to_export:
|
|
name = entry["staging_path"]
|
|
if name in seen:
|
|
duplicates.add(name)
|
|
seen.add(name)
|
|
if duplicates:
|
|
sys.exit(f"Duplicate export filenames given: {', '.join(duplicates)}, failing export")
|
|
|
|
|
|
def write_if_not_exists(file_sources, target_uri, real_data_path):
|
|
file_source_path = file_sources.get_file_source_path(target_uri)
|
|
if os.path.exists(file_source_path.path):
|
|
print(f'Error: File "{file_source_path.path}" already exists. Skipping.')
|
|
return 1
|
|
file_source = file_source_path.file_source
|
|
file_source.write_from(file_source_path.path, real_data_path)
|
|
|
|
|
|
def get_directory_uri(args):
|
|
directory_uri = args.directory_uri
|
|
if not directory_uri:
|
|
inputs = json.load(open(args.inputs))
|
|
directory_uri = inputs["d_uri"]
|
|
if not directory_uri.endswith("/"):
|
|
directory_uri = f"{directory_uri}/"
|
|
return directory_uri
|
|
|
|
|
|
def main(argv=None):
|
|
if argv is None:
|
|
argv = sys.argv[1:]
|
|
args = _parser().parse_args(argv)
|
|
exit_code = 0
|
|
file_sources = get_file_sources(args.file_sources)
|
|
directory_uri = get_directory_uri(args)
|
|
export_metadata_files = args.export_metadata_files
|
|
with open(args.files_to_export) as f:
|
|
files_to_export = json.load(f)
|
|
counter = 0
|
|
check_for_duplicate_name(files_to_export)
|
|
for entry in files_to_export:
|
|
name = entry["staging_path"]
|
|
real_data_path = entry["source_path"]
|
|
target_uri = f"{directory_uri}{name}"
|
|
if write_if_not_exists(file_sources, target_uri, real_data_path):
|
|
exit_code = 1
|
|
if export_metadata_files:
|
|
metadata_files = entry.get("metadata_files", [])
|
|
for metadata_file in metadata_files:
|
|
metadata_file_uri = f"{directory_uri}{metadata_file['staging_path']}"
|
|
if write_if_not_exists(file_sources, metadata_file_uri, metadata_file["source_path"]):
|
|
exit_code = 1
|
|
counter += 1
|
|
print(f"{counter} out of {len(files_to_export)} files have been exported.\n")
|
|
sys.exit(exit_code)
|
|
|
|
|
|
def _parser():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--inputs", type=str, help="galaxy inputs")
|
|
# post 2026.XX drop the directory-uri argument and just assume inputs always contains d_uri.
|
|
parser.add_argument("--directory-uri", type=str, help="directory target URI")
|
|
parser.add_argument("--file-sources", type=str, help="file sources json")
|
|
parser.add_argument("--files-to-export", type=str, help="files to export")
|
|
parser.add_argument("--export-metadata-files", type=bool, help="export metadata files", default=False)
|
|
return parser
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|