mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-08-31 01:02:04 +08:00
paperbenni
bec3d3ad9c
Drop Python 3.8 support in 5 Pulsar-compatible packages (job_metrics, tool_util, tool_util_models, util, objectstore) and run pyupgrade --py310-plus on their source code. - Bump requires-python from >=3.8 to >=3.10 in all 5 packages - Remove Python 3.8 and 3.9 classifiers - Remove ruff per-file-ignores for UP rules on these paths - Remove backports.zoneinfo conditional dependency - Remove pydyf<0.11 pin from conditional-requirements.txt - Update Makefile pyupgrade target (remove PY38_PYUPGRADE_PATHS) - Update CI workflow to test with Python 3.10 instead of 3.8 Clean up unused deprecated typing imports after pyupgrade Remove now-unused typing imports (Dict, List, Optional, Set, Tuple, Type, Union) that became dead after pyupgrade --py310-plus converted annotations to use built-in types and | syntax. Also run ruff check --fix --select=UP007,UP045 across the entire codebase to convert remaining Optional[X] -> X | None and Union[X, Y] -> X | Y patterns. Enable ruff UP007/UP045 for Python 3.10 union syntax Remove UP007 (Union[X,Y] -> X | Y) and UP045 (Optional[X] -> X | None) from the ruff ignore list and convert all type aliases across the codebase. These rules were deferred while Python 3.9 was supported; requires-python is now >=3.10. A custom script was used because neither ruff --fix nor pyupgrade --py310-plus converts Optional[X]/Union[X,Y] in type alias positions (e.g. X = Union[A, B]) — they only handle annotation positions (e.g. def f(x: Optional[int])). All 167 violations were module-level type aliases. A few edge cases were fixed manually: single-element Union[X,], typing.Union qualified refs, runtime Optional[type] calls, and Annotated[Optional[...]] pydantic fields. Fix UP007 autofix regression with string forward reference type aliases Commit fa6bd955a0 enabled ruff UP007/UP045 and auto-fixed module-level type aliases using Union with string forward references, producing invalid 'str | str' expressions. This was a known ruff bug (charliermarsh/ruff#826) that has since been fixed in later ruff versions, but this codebase was converted before the fix was in place. Revert to Union syntax and restore TYPE_CHECKING imports that ruff's TCH rule cleaned up as a side effect when it thought the forward references were unused.
101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
#!/usr/bin/env python
|
|
# Retrieves data from external data source applications and stores in a dataset file.
|
|
# Data source application parameters are temporarily stored in the dataset file.
|
|
import json
|
|
import os
|
|
import sys
|
|
from urllib.parse import (
|
|
urlencode,
|
|
urlparse,
|
|
)
|
|
from urllib.request import (
|
|
Request,
|
|
urlopen,
|
|
)
|
|
|
|
from galaxy.datatypes import sniff
|
|
from galaxy.datatypes.registry import Registry
|
|
from galaxy.util import (
|
|
DEFAULT_SOCKET_TIMEOUT,
|
|
get_charset_from_http_headers,
|
|
stream_to_open_named_file,
|
|
)
|
|
from galaxy.util.user_agent import get_default_headers
|
|
|
|
GALAXY_PARAM_PREFIX = "GALAXY"
|
|
GALAXY_ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
|
|
GALAXY_DATATYPES_CONF_FILE = os.path.join(GALAXY_ROOT_DIR, "datatypes_conf.xml")
|
|
|
|
|
|
def __main__():
|
|
if len(sys.argv) >= 3:
|
|
max_file_size = int(sys.argv[2])
|
|
else:
|
|
max_file_size = 0
|
|
|
|
with open(sys.argv[1]) as fh:
|
|
params = json.load(fh)
|
|
|
|
out_data_name = params["output_data"][0]["out_data_name"]
|
|
|
|
URL = params["param_dict"].get("URL", None) # using exactly URL indicates that only one dataset is being downloaded
|
|
URL_method = params["param_dict"].get("URL_method", "get")
|
|
|
|
datatypes_registry = Registry()
|
|
datatypes_registry.load_datatypes(
|
|
root_dir=params["job_config"]["GALAXY_ROOT_DIR"],
|
|
config=params["job_config"]["GALAXY_DATATYPES_CONF_FILE"],
|
|
)
|
|
|
|
for data_dict in params["output_data"]:
|
|
cur_filename = data_dict["file_name"]
|
|
cur_URL = params["param_dict"].get("{}|{}|URL".format(GALAXY_PARAM_PREFIX, data_dict["out_data_name"]), URL)
|
|
if not cur_URL or urlparse(cur_URL).scheme not in ("http", "https", "ftp"):
|
|
open(cur_filename, "w").write("")
|
|
sys.exit("The remote data source application has not sent back a URL parameter in the request.")
|
|
|
|
# The following calls to urlopen() will use the above default timeout
|
|
headers = get_default_headers()
|
|
try:
|
|
if URL_method == "get":
|
|
req = Request(cur_URL, headers=headers)
|
|
elif URL_method == "post":
|
|
data = urlencode(params["param_dict"]["incoming_request_params"]).encode("utf-8")
|
|
req = Request(cur_URL, data=data, headers=headers)
|
|
else:
|
|
raise Exception("Unknown URL_method specified: %s" % URL_method)
|
|
page = urlopen(req, timeout=DEFAULT_SOCKET_TIMEOUT)
|
|
except Exception as e:
|
|
sys.exit("The remote data source application may be off line, please try again later. Error: %s" % str(e))
|
|
if max_file_size:
|
|
file_size = int(page.info().get("Content-Length", 0))
|
|
if file_size > max_file_size:
|
|
sys.exit(
|
|
"The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server."
|
|
% (file_size, max_file_size)
|
|
)
|
|
try:
|
|
cur_filename = stream_to_open_named_file(
|
|
page,
|
|
os.open(cur_filename, os.O_WRONLY | os.O_TRUNC | os.O_CREAT),
|
|
cur_filename,
|
|
source_encoding=get_charset_from_http_headers(page.headers),
|
|
)
|
|
except Exception as e:
|
|
sys.exit(f"Unable to fetch {cur_URL}:\n{e}")
|
|
|
|
# here import checks that upload tool performs
|
|
try:
|
|
ext = sniff.handle_uploaded_dataset_file(cur_filename, datatypes_registry, ext=data_dict["ext"])
|
|
except Exception as e:
|
|
sys.exit(str(e))
|
|
|
|
tool_provided_metadata = {out_data_name: {"ext": ext}}
|
|
|
|
with open(params["job_config"]["TOOL_PROVIDED_JOB_METADATA_FILE"], "w") as json_file:
|
|
json.dump(tool_provided_metadata, json_file)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
__main__()
|