diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 7533ad9e87d..ba8a7552395 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -12,8 +12,8 @@ import shutil import struct import sys import tempfile -import urllib.request import zipfile +from functools import partial from typing import ( Dict, IO, @@ -25,13 +25,11 @@ from typing import ( from typing_extensions import Protocol from galaxy import util -from galaxy.files import ConfiguredFileSources -from galaxy.files.uris import stream_to_file +from galaxy.files.uris import stream_url_to_file as files_stream_url_to_file from galaxy.util import ( compression_utils, file_reader, is_binary, - stream_to_open_named_file, ) from galaxy.util.checkers import ( check_html, @@ -64,22 +62,7 @@ def sniff_with_cls(cls, fname): return False -def stream_url_to_file(path: str, file_sources: Optional[ConfiguredFileSources] = None): - prefix = "url_paste" - if file_sources and file_sources.looks_like_uri(path): - file_source_path = file_sources.get_file_source_path(path) - with tempfile.NamedTemporaryFile(prefix=prefix, delete=False) as temp: - temp_name = temp.name - file_source_path.file_source.realize_to(file_source_path.path, temp_name) - return temp_name - else: - page = urllib.request.urlopen( - path, timeout=util.DEFAULT_SOCKET_TIMEOUT - ) # page will be .close()ed in stream_to_file - temp_name = stream_to_file( - page, prefix=prefix, source_encoding=util.get_charset_from_http_headers(page.headers) - ) - return temp_name +stream_url_to_file = partial(files_stream_url_to_file, prefix="gx_url_paste") def handle_composite_file(datatype, src_path, extra_files, name, is_binary, tmp_dir, tmp_prefix, upload_opts): diff --git a/lib/galaxy/files/uris.py b/lib/galaxy/files/uris.py index c88ff7d47d4..5d571844672 100644 --- a/lib/galaxy/files/uris.py +++ b/lib/galaxy/files/uris.py @@ -1,9 +1,13 @@ import ipaddress import logging +import os import socket import tempfile +import urllib.request from typing import ( List, + Optional, + TYPE_CHECKING, Union, ) from urllib.parse import urlparse @@ -13,14 +17,45 @@ from galaxy.exceptions import ( ConfigDoesNotAllowException, ) from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + get_charset_from_http_headers, stream_to_open_named_file, unicodify, ) +if TYPE_CHECKING: + from galaxy.files import ConfiguredFileSources + log = logging.getLogger(__name__) +def stream_url_to_str( + path: str, file_sources: Optional["ConfiguredFileSources"] = None, prefix: str = "gx_file_stream" +) -> str: + tmp_file = stream_url_to_file(path, file_sources=file_sources, prefix=prefix) + try: + with open(tmp_file, "r") as f: + return f.read() + finally: + os.remove(tmp_file) + + +def stream_url_to_file( + path: str, file_sources: Optional["ConfiguredFileSources"] = None, prefix: str = "gx_file_stream" +) -> str: + if file_sources and file_sources.looks_like_uri(path): + file_source_path = file_sources.get_file_source_path(path) + with tempfile.NamedTemporaryFile(prefix=prefix, delete=False) as temp: + temp_name = temp.name + file_source_path.file_source.realize_to(file_source_path.path, temp_name) + return temp_name + else: + page = urllib.request.urlopen(path, timeout=DEFAULT_SOCKET_TIMEOUT) # page will be .close()ed in stream_to_file + temp_name = stream_to_file(page, prefix=prefix, source_encoding=get_charset_from_http_headers(page.headers)) + return temp_name + + def stream_to_file(stream, suffix="", prefix="", dir=None, text=False, **kwd): """Writes a stream to a temporary file, returns the temporary file's name""" fd, temp_name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 84bfe100fb5..4029a325691 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -18,7 +18,10 @@ from webob.compat import cgi_FieldStorage from galaxy import util from galaxy.exceptions import RequestParameterInvalidException -from galaxy.files.uris import stream_to_file, validate_non_local +from galaxy.files.uris import ( + stream_to_file, + validate_non_local, +) from galaxy.model import ( FormDefinition, LibraryDataset, diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index fe988442349..a3444cf8b19 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -21,7 +21,10 @@ from galaxy.datatypes.upload_util import ( handle_upload, UploadProblemException, ) -from galaxy.files.uris import stream_to_file +from galaxy.files.uris import ( + stream_to_file, + stream_url_to_file, +) from galaxy.util import ( in_directory, safe_makedirs, @@ -423,7 +426,7 @@ def _has_src_to_path(upload_config, item, is_dataset=False) -> Tuple[str, str]: if src == "url": url = item.get("url") try: - path = sniff.stream_url_to_file(url, file_sources=get_file_sources(upload_config.working_directory)) + path = stream_url_to_file(url, file_sources=get_file_sources(upload_config.working_directory)) except Exception as e: raise Exception(f"Failed to fetch url {url}. {str(e)}") diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index cfc5bc23110..8fbd0a3aa75 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -13,16 +13,16 @@ import os import tarfile from base64 import b64decode -from galaxy.datatypes import sniff +from galaxy.files import ConfiguredFileSources +from galaxy.files.uris import stream_url_to_file # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow(2, 30) -def get_file_sources(file_sources_path): +def get_file_sources(file_sources_path) -> ConfiguredFileSources: assert os.path.exists(file_sources_path), f"file sources path [{file_sources_path}] does not exist" - from galaxy.files import ConfiguredFileSources with open(file_sources_path) as f: file_sources_as_dict = json.load(f) @@ -63,7 +63,9 @@ def main(options, args): # Get archive from URL. if is_url: - archive_file = sniff.stream_url_to_file(archive_source, file_sources=get_file_sources(options.file_sources)) + archive_file = stream_url_to_file( + archive_source, file_sources=get_file_sources(options.file_sources), prefix="gx_history_archive" + ) elif is_file: archive_file = archive_source diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index e8266884b52..263644d1fc0 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -13,7 +13,6 @@ from typing import ( Optional, ) -import requests from fastapi import ( Body, Path, @@ -29,7 +28,10 @@ from galaxy import ( model, util, ) -from galaxy.files.uris import validate_uri_access +from galaxy.files.uris import ( + stream_url_to_str, + validate_uri_access, +) from galaxy.managers.context import ProvidesUserContext from galaxy.managers.jobs import ( fetch_job_states, @@ -319,7 +321,9 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U archive_data = self.app.trs_proxy.get_version_descriptor(trs_server, trs_tool_id, trs_version_id) else: try: - archive_data = requests.get(archive_source, timeout=util.DEFAULT_SOCKET_TIMEOUT).text + archive_data = stream_url_to_str( + archive_source, trans.app.file_sources, prefix="gx_workflow_download" + ) import_source = "URL" except Exception: raise exceptions.MessageException(f"Failed to open URL '{escape(archive_source)}'.") diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index 7208446cf59..4df9eb35e62 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -19,6 +19,7 @@ from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, get_charset_from_http_headers, + stream_to_open_named_file, ) GALAXY_PARAM_PREFIX = "GALAXY" @@ -107,7 +108,7 @@ def __main__(): % (file_size, max_file_size) ) try: - cur_filename = sniff.stream_to_open_named_file( + cur_filename = stream_to_open_named_file( page, os.open(cur_filename, os.O_WRONLY | os.O_CREAT), cur_filename,