From 37892dcabcafe37e65e35ea52059b707189a35ba Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 20 May 2022 14:47:07 -0400 Subject: [PATCH] Refactor sniff.stream_url_to_file to galaxy.files.uris. The value of this can be seen for instance in unpack_tar_gz_archive - which should have nothing to do with datatypes and now doesn't need to load the whole datatypes module. Use this in workflows API to allow downloading workflow files from configured URIs. --- lib/galaxy/datatypes/sniff.py | 23 ++---------- lib/galaxy/files/uris.py | 35 +++++++++++++++++++ lib/galaxy/tools/actions/upload_common.py | 5 ++- lib/galaxy/tools/data_fetch.py | 7 ++-- .../tools/imp_exp/unpack_tar_gz_archive.py | 10 +++--- lib/galaxy/webapps/galaxy/api/workflows.py | 10 ++++-- tools/data_source/data_source.py | 3 +- 7 files changed, 62 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 7533ad9e87d..ba8a7552395 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -12,8 +12,8 @@ import shutil import struct import sys import tempfile -import urllib.request import zipfile +from functools import partial from typing import ( Dict, IO, @@ -25,13 +25,11 @@ from typing import ( from typing_extensions import Protocol from galaxy import util -from galaxy.files import ConfiguredFileSources -from galaxy.files.uris import stream_to_file +from galaxy.files.uris import stream_url_to_file as files_stream_url_to_file from galaxy.util import ( compression_utils, file_reader, is_binary, - stream_to_open_named_file, ) from galaxy.util.checkers import ( check_html, @@ -64,22 +62,7 @@ def sniff_with_cls(cls, fname): return False -def stream_url_to_file(path: str, file_sources: Optional[ConfiguredFileSources] = None): - prefix = "url_paste" - if file_sources and file_sources.looks_like_uri(path): - file_source_path = file_sources.get_file_source_path(path) - with tempfile.NamedTemporaryFile(prefix=prefix, delete=False) as temp: - temp_name = temp.name - file_source_path.file_source.realize_to(file_source_path.path, temp_name) - return temp_name - else: - page = urllib.request.urlopen( - path, timeout=util.DEFAULT_SOCKET_TIMEOUT - ) # page will be .close()ed in stream_to_file - temp_name = stream_to_file( - page, prefix=prefix, source_encoding=util.get_charset_from_http_headers(page.headers) - ) - return temp_name +stream_url_to_file = partial(files_stream_url_to_file, prefix="gx_url_paste") def handle_composite_file(datatype, src_path, extra_files, name, is_binary, tmp_dir, tmp_prefix, upload_opts): diff --git a/lib/galaxy/files/uris.py b/lib/galaxy/files/uris.py index c88ff7d47d4..5d571844672 100644 --- a/lib/galaxy/files/uris.py +++ b/lib/galaxy/files/uris.py @@ -1,9 +1,13 @@ import ipaddress import logging +import os import socket import tempfile +import urllib.request from typing import ( List, + Optional, + TYPE_CHECKING, Union, ) from urllib.parse import urlparse @@ -13,14 +17,45 @@ from galaxy.exceptions import ( ConfigDoesNotAllowException, ) from galaxy.util import ( + DEFAULT_SOCKET_TIMEOUT, + get_charset_from_http_headers, stream_to_open_named_file, unicodify, ) +if TYPE_CHECKING: + from galaxy.files import ConfiguredFileSources + log = logging.getLogger(__name__) +def stream_url_to_str( + path: str, file_sources: Optional["ConfiguredFileSources"] = None, prefix: str = "gx_file_stream" +) -> str: + tmp_file = stream_url_to_file(path, file_sources=file_sources, prefix=prefix) + try: + with open(tmp_file, "r") as f: + return f.read() + finally: + os.remove(tmp_file) + + +def stream_url_to_file( + path: str, file_sources: Optional["ConfiguredFileSources"] = None, prefix: str = "gx_file_stream" +) -> str: + if file_sources and file_sources.looks_like_uri(path): + file_source_path = file_sources.get_file_source_path(path) + with tempfile.NamedTemporaryFile(prefix=prefix, delete=False) as temp: + temp_name = temp.name + file_source_path.file_source.realize_to(file_source_path.path, temp_name) + return temp_name + else: + page = urllib.request.urlopen(path, timeout=DEFAULT_SOCKET_TIMEOUT) # page will be .close()ed in stream_to_file + temp_name = stream_to_file(page, prefix=prefix, source_encoding=get_charset_from_http_headers(page.headers)) + return temp_name + + def stream_to_file(stream, suffix="", prefix="", dir=None, text=False, **kwd): """Writes a stream to a temporary file, returns the temporary file's name""" fd, temp_name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 84bfe100fb5..4029a325691 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -18,7 +18,10 @@ from webob.compat import cgi_FieldStorage from galaxy import util from galaxy.exceptions import RequestParameterInvalidException -from galaxy.files.uris import stream_to_file, validate_non_local +from galaxy.files.uris import ( + stream_to_file, + validate_non_local, +) from galaxy.model import ( FormDefinition, LibraryDataset, diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index fe988442349..a3444cf8b19 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -21,7 +21,10 @@ from galaxy.datatypes.upload_util import ( handle_upload, UploadProblemException, ) -from galaxy.files.uris import stream_to_file +from galaxy.files.uris import ( + stream_to_file, + stream_url_to_file, +) from galaxy.util import ( in_directory, safe_makedirs, @@ -423,7 +426,7 @@ def _has_src_to_path(upload_config, item, is_dataset=False) -> Tuple[str, str]: if src == "url": url = item.get("url") try: - path = sniff.stream_url_to_file(url, file_sources=get_file_sources(upload_config.working_directory)) + path = stream_url_to_file(url, file_sources=get_file_sources(upload_config.working_directory)) except Exception as e: raise Exception(f"Failed to fetch url {url}. {str(e)}") diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index cfc5bc23110..8fbd0a3aa75 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -13,16 +13,16 @@ import os import tarfile from base64 import b64decode -from galaxy.datatypes import sniff +from galaxy.files import ConfiguredFileSources +from galaxy.files.uris import stream_url_to_file # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow(2, 30) -def get_file_sources(file_sources_path): +def get_file_sources(file_sources_path) -> ConfiguredFileSources: assert os.path.exists(file_sources_path), f"file sources path [{file_sources_path}] does not exist" - from galaxy.files import ConfiguredFileSources with open(file_sources_path) as f: file_sources_as_dict = json.load(f) @@ -63,7 +63,9 @@ def main(options, args): # Get archive from URL. if is_url: - archive_file = sniff.stream_url_to_file(archive_source, file_sources=get_file_sources(options.file_sources)) + archive_file = stream_url_to_file( + archive_source, file_sources=get_file_sources(options.file_sources), prefix="gx_history_archive" + ) elif is_file: archive_file = archive_source diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index e8266884b52..263644d1fc0 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -13,7 +13,6 @@ from typing import ( Optional, ) -import requests from fastapi import ( Body, Path, @@ -29,7 +28,10 @@ from galaxy import ( model, util, ) -from galaxy.files.uris import validate_uri_access +from galaxy.files.uris import ( + stream_url_to_str, + validate_uri_access, +) from galaxy.managers.context import ProvidesUserContext from galaxy.managers.jobs import ( fetch_job_states, @@ -319,7 +321,9 @@ class WorkflowsAPIController(BaseGalaxyAPIController, UsesStoredWorkflowMixin, U archive_data = self.app.trs_proxy.get_version_descriptor(trs_server, trs_tool_id, trs_version_id) else: try: - archive_data = requests.get(archive_source, timeout=util.DEFAULT_SOCKET_TIMEOUT).text + archive_data = stream_url_to_str( + archive_source, trans.app.file_sources, prefix="gx_workflow_download" + ) import_source = "URL" except Exception: raise exceptions.MessageException(f"Failed to open URL '{escape(archive_source)}'.") diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index 7208446cf59..4df9eb35e62 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -19,6 +19,7 @@ from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, get_charset_from_http_headers, + stream_to_open_named_file, ) GALAXY_PARAM_PREFIX = "GALAXY" @@ -107,7 +108,7 @@ def __main__(): % (file_size, max_file_size) ) try: - cur_filename = sniff.stream_to_open_named_file( + cur_filename = stream_to_open_named_file( page, os.open(cur_filename, os.O_WRONLY | os.O_CREAT), cur_filename,