Merge branch 'release_20.01' into dev

doc/source/admin/galaxy_options.rst

@@ -3622,6 +3622,120 @@
 :Type: str
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``enable_beta_markdown_export``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Enable export of Galaxy Markdown documents (pages and workflow
+    reports) to PDF. Requires manual installation and setup of
+    weasyprint (latest version available for Python 2.7 is 0.42).
+:Default: ``false``
+:Type: bool
+
+
+~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_css``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    CSS file to apply to all Markdown exports to PDF - currently used
+    by WeasyPrint during rendering an HTML export of the document to
+    PDF.
+:Default: ``markdown_export.css``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_css_pages``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    CSS file to apply to "Galaxy Page" exports to PDF. Generally
+    prefer  markdown_export_css, but this is here for deployments that
+    would like to tailor different kinds of exports.
+:Default: ``markdown_export_pages.css``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_css_invocation_reports``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    CSS file to apply to invocation report exports to PDF. Generally
+    prefer  markdown_export_css, but this is here for deployments that
+    would like to tailor different kinds of exports.
+:Default: ``markdown_export_invocation_reports.css``
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_prologue``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Prologue Markdown/HTML to apply to markdown exports to PDF.
+    Allowing branded headers.
+:Default: ````
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_epilogue``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Prologue Markdown/HTML to apply to markdown exports to PDF.
+    Allowing branded footers.
+:Default: ````
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_prologue_pages``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Alternative to markdown_export_prologue that applies just to page
+    exports.
+:Default: ````
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_prologue_invocation_reports``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Alternative to markdown_export_prologue that applies just to
+    invocation report exports.
+:Default: ````
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_epilogue_pages``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Alternative to markdown_export_epilogue that applies just to page
+    exports.
+:Default: ````
+:Type: str
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``markdown_export_epilogue_invocation_reports``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Alternative to markdown_export_epilogue that applies just to
+    invocation report exports.
+:Default: ````
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``job_resource_params_file``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~

lib/galaxy/config/sample/galaxy.yml.sample

@@ -1785,6 +1785,49 @@ galaxy:
   # environment individually.
   #environment_setup_file: null
 
+  # Enable export of Galaxy Markdown documents (pages and workflow
+  # reports) to PDF. Requires manual installation and setup of
+  # weasyprint (latest version available for Python 2.7 is 0.42).
+  #enable_beta_markdown_export: false
+
+  # CSS file to apply to all Markdown exports to PDF - currently used by
+  # WeasyPrint during rendering an HTML export of the document to PDF.
+  #markdown_export_css: markdown_export.css
+
+  # CSS file to apply to "Galaxy Page" exports to PDF. Generally prefer
+  # markdown_export_css, but this is here for deployments that would
+  # like to tailor different kinds of exports.
+  #markdown_export_css_pages: markdown_export_pages.css
+
+  # CSS file to apply to invocation report exports to PDF. Generally
+  # prefer  markdown_export_css, but this is here for deployments that
+  # would like to tailor different kinds of exports.
+  #markdown_export_css_invocation_reports: markdown_export_invocation_reports.css
+
+  # Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing
+  # branded headers.
+  #markdown_export_prologue: ''
+
+  # Prologue Markdown/HTML to apply to markdown exports to PDF. Allowing
+  # branded footers.
+  #markdown_export_epilogue: ''
+
+  # Alternative to markdown_export_prologue that applies just to page
+  # exports.
+  #markdown_export_prologue_pages: ''
+
+  # Alternative to markdown_export_prologue that applies just to
+  # invocation report exports.
+  #markdown_export_prologue_invocation_reports: ''
+
+  # Alternative to markdown_export_epilogue that applies just to page
+  # exports.
+  #markdown_export_epilogue_pages: ''
+
+  # Alternative to markdown_export_epilogue that applies just to
+  # invocation report exports.
+  #markdown_export_epilogue_invocation_reports: ''
+
   # Optional file containing job resource data entry fields definition.
   # These fields will be presented to users in the tool forms and allow
   # them to overwrite default job resources such as number of

lib/galaxy/datatypes/binary.py

@@ -1119,8 +1119,8 @@ class Cool(H5):
         if super(Cool, self).sniff(filename):
             keys = ['chroms', 'bins', 'pixels', 'indexes']
             with h5py.File(filename, 'r') as handle:
-                fmt = handle.attrs.get('format')
-                url = handle.attrs.get('format-url')
+                fmt = util.unicodify(handle.attrs.get('format'))
+                url = util.unicodify(handle.attrs.get('format-url'))
                 if fmt == MAGIC or url == URL:
                     if not all(name in handle.keys() for name in keys):
                         return False
@@ -1179,8 +1179,8 @@ class MCool(H5):
                     return False
                 res0 = list(handle['resolutions'].keys())[0]
                 keys = ['chroms', 'bins', 'pixels', 'indexes']
-                fmt = handle['resolutions'][res0].attrs.get('format')
-                url = handle['resolutions'][res0].attrs.get('format-url')
+                fmt = util.unicodify(handle['resolutions'][res0].attrs.get('format'))
+                url = util.unicodify(handle['resolutions'][res0].attrs.get('format-url'))
                 if fmt == MAGIC or url == URL:
                     if not all(name in handle['resolutions'][res0].keys() for name in keys):
                         return False

lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt

@@ -20,7 +20,7 @@ imagesize==1.1.0
 importlib-metadata==1.2.0 ; python_version < '3.8'
 jinja2==2.10.3
 lxml==4.4.2
-markdown==3.0.1
+markdown==3.1.1
 markupsafe==1.1.1
 mirakuru==1.1.0
 mock==3.0.5

lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt

@@ -82,6 +82,7 @@ kombu==4.6.6
 lockfile==0.12.2
 lxml==4.4.2
 mako==1.1.0
+markdown==3.1.1
 markupsafe==1.1.1
 mercurial==5.2
 mistune==0.8.4

lib/galaxy/jobs/__init__.py

@@ -1034,11 +1034,11 @@ class JobWrapper(HasResourceParameters):
         return False
 
     def get_version_string_path(self):
-        return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME))
+        return os.path.abspath(os.path.join(self.working_directory, "outputs", COMMAND_VERSION_FILENAME))
 
-    # TODO: Remove in Galaxy 20.XX, for running jobs at GX upgrade
+    # TODO: Remove in Galaxy 21.XX, for running jobs at GX upgrade
     def get_version_string_path_legacy(self):
-        return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id))
+        return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME))
 
     def __prepare_upload_paramfile(self, tool_evaluator):
         """Special case paramfile handling for the upload tool. Moves the paramfile to the working directory

lib/galaxy/model/__init__.py

@@ -659,7 +659,7 @@ class TaskMetricNumeric(BaseJobMetric, RepresentById):
 
 class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
     dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version']
-    dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version']
+    dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version']
 
     """
     A job represents a request to run a tool given input datasets, tool
@@ -1132,6 +1132,12 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
             config_value = default
         return config_value
 
+    @property
+    def command_version(self):
+        # TODO: make actual database property and track properly - we should be recording this on the job and not on the datasets
+        for dataset_assoc in self.output_datasets:
+            return dataset_assoc.dataset.tool_version
+
 
 class Task(JobLike, RepresentById):
     """

lib/galaxy/tool_util/deps/container_classes.py

@@ -179,7 +179,7 @@ class HasDockerLikeVolumes(object):
             if self.job_info.tool_directory:
                 defaults += ",$tool_directory:default_ro"
             if self.job_info.job_directory:
-                defaults += ",$job_directory:default_ro"
+                defaults += ",$job_directory:default_ro,$job_directory/outputs:rw"
             if self.job_info.tmp_directory is not None:
                 defaults += ",$tmp_directory:rw"
             if self.job_info.home_directory is not None:

lib/galaxy/tool_util/deps/mulled/util.py

@@ -8,7 +8,6 @@ import re
 import sys
 import tarfile
 import threading
-import time
 from io import BytesIO
 
 import packaging.version
@@ -293,13 +292,13 @@ def split_container_name(name):
 class PrintProgress(object):
     def __init__(self):
         self.thread = threading.Thread(target=self.progress)
-        self.stop = False
+        self.stop = threading.Event()
 
     def progress(self):
-        while not self.stop:
+        while not self.stop.is_set():
             print(".", end="")
             sys.stdout.flush()
-            time.sleep(60)
+            self.stop.wait(60)
         print("")
 
     def __enter__(self):
@@ -307,7 +306,7 @@ class PrintProgress(object):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.stop = True
+        self.stop.set()
         self.thread.join()
 
 

lib/galaxy/tool_util/parser/xml.py

@@ -531,6 +531,7 @@ def _test_elem_to_dict(test_elem, i):
         inputs=__parse_input_elems(test_elem, i),
         expect_num_outputs=test_elem.get("expect_num_outputs"),
         command=__parse_assert_list_from_elem(test_elem.find("assert_command")),
+        command_version=__parse_assert_list_from_elem(test_elem.find("assert_command_version")),
         stdout=__parse_assert_list_from_elem(test_elem.find("assert_stdout")),
         stderr=__parse_assert_list_from_elem(test_elem.find("assert_stderr")),
         expect_exit_code=test_elem.get("expect_exit_code"),

lib/galaxy/tool_util/verify/interactor.py

@@ -901,6 +901,7 @@ def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_
 
     other_checks = {
         "command_line": "Command produced by the job",
+        "command_version": "Tool version indicated during job execution",
         "stdout": "Standard output of the job",
         "stderr": "Standard error of the job",
     }
@@ -1064,6 +1065,7 @@ class ToolTestDescription(object):
 
         self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", []))
         self.command_line = processed_test_dict.get("command_line", None)
+        self.command_version = processed_test_dict.get("command_version", None)
         self.stdout = processed_test_dict.get("stdout", None)
         self.stderr = processed_test_dict.get("stderr", None)
         self.expect_exit_code = processed_test_dict.get("expect_exit_code", None)
@@ -1089,6 +1091,7 @@ class ToolTestDescription(object):
             "output_collections": map(lambda o: o.to_dict(), self.output_collections),
             "num_outputs": self.num_outputs,
             "command_line": self.command_line,
+            "command_version": self.command_version,
             "stdout": self.stdout,
             "stderr": self.stderr,
             "expect_exit_code": self.expect_exit_code,

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -851,6 +851,15 @@ $assertions
           <xs:documentation xml:lang="en">Describe assertions about the job's
 standard error.
 
+$assertions
+</xs:documentation>
+        </xs:annotation>
+      </xs:element>
+      <xs:element name="assert_command_version" type="TestAssertions">
+        <xs:annotation>
+          <xs:documentation xml:lang="en">Describe assertions about the job's
+command version.
+
 $assertions
 </xs:documentation>
         </xs:annotation>

lib/galaxy/tools/test.py

@@ -50,6 +50,7 @@ def description_from_tool_object(tool, test_index, raw_test_dict):
             "output_collections": raw_test_dict["output_collections"],
             "num_outputs": num_outputs,
             "command_line": raw_test_dict.get("command", None),
+            "command_version": raw_test_dict.get("command_version", None),
             "stdout": raw_test_dict.get("stdout", None),
             "stderr": raw_test_dict.get("stderr", None),
             "expect_exit_code": raw_test_dict.get("expect_exit_code", None),

test/functional/tools/job_properties.xml

@@ -2,6 +2,7 @@
     <stdio>
         <exit_code range="127" level="fatal" description="Failing exit code." />
     </stdio>
+    <version_command>echo 'v1.1'</version_command>
     <command><![CDATA[
         #if $thebool
             echo "The bool is true" &&
@@ -44,6 +45,9 @@
             <assert_stderr>
                 <has_line line="The bool is really true" />
             </assert_stderr>
+            <assert_command_version>
+                <has_text text="v1.1" />
+            </assert_command_version>
         </test>
         <test expect_exit_code="2">
             <param name="thebool" value="false" />

test/unit/jobs/test_job_wrapper.py

@@ -53,7 +53,7 @@ class BaseWrapperTestCase(UsesApp):
     def test_version_path(self):
         wrapper = self._wrapper()
         version_path = wrapper.get_version_string_path_legacy()
-        expected_path = os.path.join(self.test_directory, "new_files", "GALAXY_VERSION_STRING_345")
+        expected_path = os.path.join(self.test_directory, "working", "COMMAND_VERSION")
         self.assertEqual(version_path, expected_path)
 
     def test_prepare_sets_command_line(self):

test/unit/tool_util/test_parsing.py

@@ -641,6 +641,18 @@ class ExpectationsTestCase(BaseLoaderTestCase):
         assert len(test_0["stdout"]) == 2
 
 
+class ExpectationsCommandVersionTestCase(BaseLoaderTestCase):
+    source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/job_properties.xml")
+    source_contents = None
+
+    def test_tests(self):
+        tests_dict = self._tool_source.parse_tests_to_dict()
+        tests = tests_dict["tests"]
+        assert len(tests) > 0
+        test_0 = tests[0]
+        assert len(test_0["command_version"]) == 1
+
+
 class QcStdioTestCase(BaseLoaderTestCase):
     source_file_name = os.path.join(galaxy_directory(), "test/functional/tools/qc_stdout.xml")
     source_contents = None