From 642c28a3e9a047274df27b25b3d3163b583bb7b3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 14 Nov 2022 10:22:28 +0100 Subject: [PATCH 1/6] Release galaxy-tool-util-22.1.5 --- packages/tool_util/HISTORY.rst | 5 +++++ packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/tool_util/HISTORY.rst b/packages/tool_util/HISTORY.rst index 4c87e73604f..673b2d76f18 100644 --- a/packages/tool_util/HISTORY.rst +++ b/packages/tool_util/HISTORY.rst @@ -5,6 +5,11 @@ History .. to_doc +22.1.5 (2022-11-14) +------------------- + +* Set test status to success on expected failure + 22.1.4 (2022-10-28) ------------------- diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index 3c74f7dfbb1..0baea8aa983 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,4 +1,4 @@ -__version__ = "22.1.5.dev0" +__version__ = "22.1.5" PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From df7712927b8480e7475b43c23c32a9e5998a885d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 14 Nov 2022 10:28:55 +0100 Subject: [PATCH 2/6] Start work on galaxy-tool-util 22.1.6 --- packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index 0baea8aa983..14db50b54c4 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,4 +1,4 @@ -__version__ = "22.1.5" +__version__ = "22.1.6.dev0" PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From 162c3b45b756fdc220a947d453919eff6df25433 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 11 Nov 2022 20:25:19 -0500 Subject: [PATCH 3/6] Specify bgzip_file as source for metadata of vcf files for igv display application --- lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml b/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml index 3c66ffd677a..e738b77c184 100644 --- a/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml +++ b/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml @@ -13,7 +13,7 @@ ${redirect_url} - + #if ($dataset.dbkey in $site_dbkeys) $site_organisms[ $site_dbkeys.index( $bgzip_file.dbkey ) ] From f053c65b1f395d8308c69e1b80f7d86f4e9c4148 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 11 Nov 2022 21:09:42 -0500 Subject: [PATCH 4/6] Only access metadata if dataset is in ok state --- lib/galaxy/datatypes/display_applications/parameters.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index c1a28fc8147..13084247b10 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -100,6 +100,8 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter): assert data, "Base dataset could not be found in values provided to DisplayApplicationDataParameter" if isinstance(data, DisplayDataValueWrapper): data = data.value + if data.state != data.states.OK: + return None if self.metadata: rval = getattr(data.metadata, self.metadata, None) assert rval, f'Unknown metadata name "{self.metadata}" provided for dataset type "{data.ext}".' @@ -109,8 +111,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter): rval = data.get_converted_files_by_type(ext) if rval: return rval - - direct_match, target_ext, converted_dataset = data.find_conversion_destination(self.formats) + direct_match, target_ext, _ = data.find_conversion_destination(self.formats) assert direct_match or target_ext is not None, f"No conversion path found for data param: {self.name}" return None return data From 7254dc9fef59def243bc54f793c9e52c270c122e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 16 Nov 2022 16:54:10 +0100 Subject: [PATCH 5/6] Also support byte-range in legacy send_file method --- .../display_applications/configs/igv/vcf.xml | 4 +-- lib/galaxy/web/framework/base.py | 27 ++++++++++++++++--- test/unit/webapps/test_send_file.py | 2 ++ 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml b/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml index e738b77c184..9e9e2c614df 100644 --- a/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml +++ b/lib/galaxy/datatypes/display_applications/configs/igv/vcf.xml @@ -13,7 +13,7 @@ ${redirect_url} - + #if ($dataset.dbkey in $site_dbkeys) $site_organisms[ $site_dbkeys.index( $bgzip_file.dbkey ) ] @@ -95,7 +95,7 @@ http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bgzip_file.qp}&genome=${qp($bgzip_file.dbkey)}&merge=true&name=${qp( ( $bgzip_file.name or $DATASET_HASH ).replace( ',', ';' ) )} - + diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 5cbcb008490..ee1693315dc 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -530,20 +530,41 @@ def send_file(start_response, trans, body): body = [b""] # Fall back on sending the file in chunks else: - body = iterate_file(body) + trans.response.headers["accept-ranges"] = "bytes" + start = None + end = None + if trans.request.range: + start = trans.request.range.start + end = trans.request.range.end + file_size = trans.response.headers["content-length"] + trans.response.headers["content-length"] = str(end - start) + trans.response.headers["content-range"] = f"bytes {start}-{end - 1}/{file_size}" + trans.response.status = 206 + body = iterate_file(body, start, end) start_response(trans.response.wsgi_status(), trans.response.wsgi_headeritems()) return body -def iterate_file(fh): +def iterate_file(fh, start=None, stop=None): """ Progressively return chunks from `file`. """ + length = None + if start: + fh.seek(start) + if stop: + length = stop - start while 1: - chunk = fh.read(CHUNK_SIZE) + read_size = CHUNK_SIZE + if length: + read_size = min(CHUNK_SIZE, length) + length -= read_size + chunk = fh.read(read_size) if not chunk: break yield chunk + if length is not None and length == 0: + break def flatten(seq): diff --git a/test/unit/webapps/test_send_file.py b/test/unit/webapps/test_send_file.py index 6d1d0a04a25..4736e370329 100644 --- a/test/unit/webapps/test_send_file.py +++ b/test/unit/webapps/test_send_file.py @@ -7,6 +7,7 @@ from fastapi.testclient import TestClient from galaxy.util.bunch import Bunch from galaxy.web.framework.base import ( + Request, Response, send_file, ) @@ -18,6 +19,7 @@ def setup_fastAPI(fh, nginx_x_accel_redirect_base=None, apache_xsendfile=None): def wsgi_application(env, start_response): trans = Bunch( response=Response(), + request=Request(env), app=Bunch( config=Bunch(nginx_x_accel_redirect_base=nginx_x_accel_redirect_base, apache_xsendfile=apache_xsendfile) ), From c80577e0b3f163446812a0f9bcfe6700327369a5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 16 Nov 2022 21:58:23 +0000 Subject: [PATCH 6/6] Fix setting ``LC_CTYPE`` variable in clean environment on macOS Fix some older tools (not using `profile` or `stdio`) failing because of the following warning printed on stderr when run locally on macOS: ``` perl: warning: Setting locale failed. perl: warning: Please check that your locale settings: LC_ALL = (unset), LC_CTYPE = "C.UTF-8", LANG = (unset) are supported and installed on your system. perl: warning: Falling back to the standard locale ("C"). ``` --- lib/galaxy/util/commands.py | 5 +++-- test/unit/util/test_commands.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 test/unit/util/test_commands.py diff --git a/lib/galaxy/util/commands.py b/lib/galaxy/util/commands.py index c4ac1668793..37df5d87489 100644 --- a/lib/galaxy/util/commands.py +++ b/lib/galaxy/util/commands.py @@ -164,7 +164,7 @@ def new_clean_env(): Returns a minimal environment to use when invoking a subprocess """ env = {} - for k in ("HOME", "PATH", "TMPDIR"): + for k in ("HOME", "LC_CTYPE", "PATH", "TMPDIR"): if k in os.environ: env[k] = os.environ[k] if "TMPDIR" not in env: @@ -173,7 +173,8 @@ def new_clean_env(): # This is needed e.g. for Python < 3.7 where # `locale.getpreferredencoding()` (also used by open() to determine the # default file encoding) would return `ANSI_X3.4-1968` without this. - env["LC_CTYPE"] = "C.UTF-8" + if not env.get("LC_CTYPE", "").endswith("UTF-8"): + env["LC_CTYPE"] = "C.UTF-8" return env diff --git a/test/unit/util/test_commands.py b/test/unit/util/test_commands.py new file mode 100644 index 00000000000..59d8a89e4a6 --- /dev/null +++ b/test/unit/util/test_commands.py @@ -0,0 +1,20 @@ +import os + +from galaxy.util.commands import new_clean_env + + +def test_new_clean_env() -> None: + saved_environ = os.environ.copy() + os.environ["FOO"] = "foo" + os.environ.pop("TMPDIR", None) + try: + clean_env = new_clean_env() + finally: + os.environ.clear() + os.environ.update(saved_environ) + assert "FOO" not in clean_env + for k in ("HOME", "PATH"): + if k in saved_environ: + assert clean_env[k] == saved_environ[k] + assert clean_env["LC_CTYPE"].endswith("UTF-8") + assert clean_env["TMPDIR"]