diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index e244b0a8480..337c64051bc 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -3622,6 +3622,120 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``enable_beta_markdown_export`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Enable export of Galaxy Markdown documents (pages and workflow + reports) to PDF. Requires manual installation and setup of + weasyprint (latest version available for Python 2.7 is 0.42). +:Default: ``false`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_css`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + CSS file to apply to all Markdown exports to PDF - currently used + by WeasyPrint during rendering an HTML export of the document to + PDF. +:Default: ``markdown_export.css`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_css_pages`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + CSS file to apply to "Galaxy Page" exports to PDF. Generally + prefer markdown_export_css, but this is here for deployments that + would like to tailor different kinds of exports. +:Default: ``markdown_export_pages.css`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_css_invocation_reports`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + CSS file to apply to invocation report exports to PDF. Generally + prefer markdown_export_css, but this is here for deployments that + would like to tailor different kinds of exports. +:Default: ``markdown_export_invocation_reports.css`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_prologue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Prologue Markdown/HTML to apply to markdown exports to PDF. + Allowing branded headers. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_epilogue`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Prologue Markdown/HTML to apply to markdown exports to PDF. + Allowing branded footers. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_prologue_pages`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_prologue that applies just to page + exports. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_prologue_invocation_reports`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_prologue that applies just to + invocation report exports. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_epilogue_pages`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_epilogue that applies just to page + exports. +:Default: ```` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``markdown_export_epilogue_invocation_reports`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Alternative to markdown_export_epilogue that applies just to + invocation report exports. +:Default: ```` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``job_resource_params_file`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 05a7666e407..eccf4c67895 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1785,6 +1785,49 @@ galaxy: # environment individually. #environment_setup_file: null + # Enable export of Galaxy Markdown documents (pages and workflow + # reports) to PDF. Requires manual installation and setup of + # weasyprint (latest version available for Python 2.7 is 0.42). + #enable_beta_markdown_export: false + + # CSS file to apply to all Markdown exports to PDF - currently used by + # WeasyPrint during rendering an HTML export of the document to PDF. + #markdown_export_css: markdown_export.css + + # CSS file to apply to "Galaxy Page" exports to PDF. Generally prefer + # markdown_export_css, but this is here for deployments that would + # like to tailor different kinds of exports. + #markdown_export_css_pages: markdown_export_pages.css + + # CSS file to apply to invocation report exports to PDF. Generally + # prefer markdown_export_css, but this is here for deployments that + # would like to tailor different kinds of exports. + #markdown_export_css_invocation_reports: markdown_export_invocation_reports.css + + # Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing + # branded headers. + #markdown_export_prologue: '' + + # Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing + # branded footers. + #markdown_export_epilogue: '' + + # Alternative to markdown_export_prologue that applies just to page + # exports. + #markdown_export_prologue_pages: '' + + # Alternative to markdown_export_prologue that applies just to + # invocation report exports. + #markdown_export_prologue_invocation_reports: '' + + # Alternative to markdown_export_epilogue that applies just to page + # exports. + #markdown_export_epilogue_pages: '' + + # Alternative to markdown_export_epilogue that applies just to + # invocation report exports. + #markdown_export_epilogue_invocation_reports: '' + # Optional file containing job resource data entry fields definition. # These fields will be presented to users in the tool forms and allow # them to overwrite default job resources such as number of diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index a8d5d73d7b1..e88bd656a2a 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1119,8 +1119,8 @@ class Cool(H5): if super(Cool, self).sniff(filename): keys = ['chroms', 'bins', 'pixels', 'indexes'] with h5py.File(filename, 'r') as handle: - fmt = handle.attrs.get('format') - url = handle.attrs.get('format-url') + fmt = util.unicodify(handle.attrs.get('format')) + url = util.unicodify(handle.attrs.get('format-url')) if fmt == MAGIC or url == URL: if not all(name in handle.keys() for name in keys): return False @@ -1179,8 +1179,8 @@ class MCool(H5): return False res0 = list(handle['resolutions'].keys())[0] keys = ['chroms', 'bins', 'pixels', 'indexes'] - fmt = handle['resolutions'][res0].attrs.get('format') - url = handle['resolutions'][res0].attrs.get('format-url') + fmt = util.unicodify(handle['resolutions'][res0].attrs.get('format')) + url = util.unicodify(handle['resolutions'][res0].attrs.get('format-url')) if fmt == MAGIC or url == URL: if not all(name in handle['resolutions'][res0].keys() for name in keys): return False diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 02c01a9bc06..d5784c22828 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -20,7 +20,7 @@ imagesize==1.1.0 importlib-metadata==1.2.0 ; python_version < '3.8' jinja2==2.10.3 lxml==4.4.2 -markdown==3.0.1 +markdown==3.1.1 markupsafe==1.1.1 mirakuru==1.1.0 mock==3.0.5 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 4214b7cec27..b7ff526367a 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -82,6 +82,7 @@ kombu==4.6.6 lockfile==0.12.2 lxml==4.4.2 mako==1.1.0 +markdown==3.1.1 markupsafe==1.1.1 mercurial==5.2 mistune==0.8.4 diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ef942469d48..9bb04908efd 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1034,11 +1034,11 @@ class JobWrapper(HasResourceParameters): return False def get_version_string_path(self): - return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) + return os.path.abspath(os.path.join(self.working_directory, "outputs", COMMAND_VERSION_FILENAME)) - # TODO: Remove in Galaxy 20.XX, for running jobs at GX upgrade + # TODO: Remove in Galaxy 21.XX, for running jobs at GX upgrade def get_version_string_path_legacy(self): - return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id)) + return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) def __prepare_upload_paramfile(self, tool_evaluator): """Special case paramfile handling for the upload tool. Moves the paramfile to the working directory diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e687b521df0..e77c9de7920 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -659,7 +659,7 @@ class TaskMetricNumeric(BaseJobMetric, RepresentById): class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] - dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] + dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version'] """ A job represents a request to run a tool given input datasets, tool @@ -1132,6 +1132,12 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): config_value = default return config_value + @property + def command_version(self): + # TODO: make actual database property and track properly - we should be recording this on the job and not on the datasets + for dataset_assoc in self.output_datasets: + return dataset_assoc.dataset.tool_version + class Task(JobLike, RepresentById): """ diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 9970334fdea..aaab9a05650 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -179,7 +179,7 @@ class HasDockerLikeVolumes(object): if self.job_info.tool_directory: defaults += ",$tool_directory:default_ro" if self.job_info.job_directory: - defaults += ",$job_directory:default_ro" + defaults += ",$job_directory:default_ro,$job_directory/outputs:rw" if self.job_info.tmp_directory is not None: defaults += ",$tmp_directory:rw" if self.job_info.home_directory is not None: diff --git a/lib/galaxy/tool_util/deps/mulled/util.py b/lib/galaxy/tool_util/deps/mulled/util.py index d005d89ffd6..57e76127180 100644 --- a/lib/galaxy/tool_util/deps/mulled/util.py +++ b/lib/galaxy/tool_util/deps/mulled/util.py @@ -8,7 +8,6 @@ import re import sys import tarfile import threading -import time from io import BytesIO import packaging.version @@ -293,13 +292,13 @@ def split_container_name(name): class PrintProgress(object): def __init__(self): self.thread = threading.Thread(target=self.progress) - self.stop = False + self.stop = threading.Event() def progress(self): - while not self.stop: + while not self.stop.is_set(): print(".", end="") sys.stdout.flush() - time.sleep(60) + self.stop.wait(60) print("") def __enter__(self): @@ -307,7 +306,7 @@ class PrintProgress(object): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.stop = True + self.stop.set() self.thread.join() diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 9aaddfa09ee..f38301762e0 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -531,6 +531,7 @@ def _test_elem_to_dict(test_elem, i): inputs=__parse_input_elems(test_elem, i), expect_num_outputs=test_elem.get("expect_num_outputs"), command=__parse_assert_list_from_elem(test_elem.find("assert_command")), + command_version=__parse_assert_list_from_elem(test_elem.find("assert_command_version")), stdout=__parse_assert_list_from_elem(test_elem.find("assert_stdout")), stderr=__parse_assert_list_from_elem(test_elem.find("assert_stderr")), expect_exit_code=test_elem.get("expect_exit_code"), diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index fc404959ee0..cac93ddd30d 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -901,6 +901,7 @@ def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_ other_checks = { "command_line": "Command produced by the job", + "command_version": "Tool version indicated during job execution", "stdout": "Standard output of the job", "stderr": "Standard error of the job", } @@ -1064,6 +1065,7 @@ class ToolTestDescription(object): self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", [])) self.command_line = processed_test_dict.get("command_line", None) + self.command_version = processed_test_dict.get("command_version", None) self.stdout = processed_test_dict.get("stdout", None) self.stderr = processed_test_dict.get("stderr", None) self.expect_exit_code = processed_test_dict.get("expect_exit_code", None) @@ -1089,6 +1091,7 @@ class ToolTestDescription(object): "output_collections": map(lambda o: o.to_dict(), self.output_collections), "num_outputs": self.num_outputs, "command_line": self.command_line, + "command_version": self.command_version, "stdout": self.stdout, "stderr": self.stderr, "expect_exit_code": self.expect_exit_code, diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 05f7355b97b..57dfedab140 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -851,6 +851,15 @@ $assertions Describe assertions about the job's standard error. +$assertions + + + + + + Describe assertions about the job's +command version. + $assertions diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index 8dc6e5d41fc..8313832d494 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -50,6 +50,7 @@ def description_from_tool_object(tool, test_index, raw_test_dict): "output_collections": raw_test_dict["output_collections"], "num_outputs": num_outputs, "command_line": raw_test_dict.get("command", None), + "command_version": raw_test_dict.get("command_version", None), "stdout": raw_test_dict.get("stdout", None), "stderr": raw_test_dict.get("stderr", None), "expect_exit_code": raw_test_dict.get("expect_exit_code", None), diff --git a/test/functional/tools/job_properties.xml b/test/functional/tools/job_properties.xml index 1bad9a7f4e2..753aa026a25 100644 --- a/test/functional/tools/job_properties.xml +++ b/test/functional/tools/job_properties.xml @@ -2,6 +2,7 @@ + echo 'v1.1' + + + diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index 50389f204ef..30289b74118 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -53,7 +53,7 @@ class BaseWrapperTestCase(UsesApp): def test_version_path(self): wrapper = self._wrapper() version_path = wrapper.get_version_string_path_legacy() - expected_path = os.path.join(self.test_directory, "new_files", "GALAXY_VERSION_STRING_345") + expected_path = os.path.join(self.test_directory, "working", "COMMAND_VERSION") self.assertEqual(version_path, expected_path) def test_prepare_sets_command_line(self): diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index da646d19894..27047fe46a2 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -641,6 +641,18 @@ class ExpectationsTestCase(BaseLoaderTestCase): assert len(test_0["stdout"]) == 2 +class ExpectationsCommandVersionTestCase(BaseLoaderTestCase): + source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/job_properties.xml") + source_contents = None + + def test_tests(self): + tests_dict = self._tool_source.parse_tests_to_dict() + tests = tests_dict["tests"] + assert len(tests) > 0 + test_0 = tests[0] + assert len(test_0["command_version"]) == 1 + + class QcStdioTestCase(BaseLoaderTestCase): source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/qc_stdout.xml") source_contents = None