Merge remote-tracking branch 'upstream/release_20.05' into dev

config/plugins/interactive_environments/common/templates/ie.mako

@@ -1,8 +1,8 @@
 <%def name="load_default_js()">
 ${h.css( 'base' ) }
-${h.js('bundled/libs.chunk',
-       'bundled/base.chunk',
-       'bundled/generic.bundled')}
+${h.dist_js('libs.chunk',
+            'base.chunk',
+            'generic.bundled')}
 </%def>
 
 <%def name="default_javascript_variables()">

lib/galaxy/jobs/__init__.py

@@ -918,7 +918,7 @@ class JobWrapper(HasResourceParameters):
             metadata_strategy_override = None
         if job.tasks:
             metadata_strategy_override = "directory"
-        self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override)
+        self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override, tool_id=job.tool_id)
 
         self.__commands_in_new_shell = True
         self.__user_system_pwent = None

lib/galaxy/metadata/__init__.py

@@ -21,11 +21,11 @@ log = getLogger(__name__)
 SET_METADATA_SCRIPT = 'from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()'
 
 
-def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None):
+def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None, tool_id=None):
     metadata_strategy = metadata_strategy_override or config.metadata_strategy
     if metadata_strategy == "legacy":
         return JobExternalOutputMetadataWrapper(job_id)
-    elif metadata_strategy == "extended":
+    elif metadata_strategy == "extended" and tool_id != "__SET_METADATA__":
         return ExtendedDirectoryMetadataGenerator(job_id)
     else:
         return PortableDirectoryMetadataGenerator(job_id)

lib/galaxy/metadata/set_metadata.py

@@ -168,7 +168,6 @@ def set_metadata_portable():
         if os.path.exists(COMMAND_VERSION_FILENAME):
             version_string = open(COMMAND_VERSION_FILENAME).read()
 
-        # TODO: handle outputs_to_working_directory?
         from galaxy.util.expressions import ExpressionContext
         job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr))
 
@@ -237,7 +236,10 @@ def set_metadata_portable():
                 dataset.set_size()
                 if 'uuid' in context:
                     dataset.dataset.uuid = context['uuid']
-                object_store.update_from_file(dataset.dataset, create=True)
+                if dataset_filename_override and dataset_filename_override != dataset.file_name:
+                    # This has to be a job with outputs_to_working_directory set.
+                    # We update the object store with the created output file.
+                    object_store.update_from_file(dataset.dataset, file_name=dataset_filename_override, create=True)
                 from galaxy.job_execution.output_collect import collect_extra_files
                 collect_extra_files(object_store, dataset, ".")
                 if galaxy.model.Job.states.ERROR == final_job_state:
@@ -265,11 +267,9 @@ def set_metadata_portable():
                     if context_key in context:
                         context_value = context[context_key]
                         setattr(dataset, context_key, context_value)
-
-                if extended_metadata_collection:
-                    export_store.add_dataset(dataset)
-                else:
-                    cPickle.dump(dataset, open(filename_out, 'wb+'))
+                # We never want to persist the external_filename.
+                dataset.dataset.external_filename = None
+                export_store.add_dataset(dataset)
             else:
                 dataset.metadata.to_JSON_dict(filename_out)  # write out results of set_meta
 

lib/galaxy/objectstore/__init__.py

@@ -368,7 +368,7 @@ class DiskObjectStore(ConcreteObjectStore):
         :param extra_dirs: Keys are string, values are directory paths.
         """
         super().__init__(config, config_dict)
-        self.file_path = config_dict.get("files_dir") or config.file_path
+        self.file_path = os.path.abspath(config_dict.get("files_dir") or config.file_path)
 
     @classmethod
     def parse_xml(clazz, config_xml):

lib/galaxy/tools/__init__.py

@@ -2547,7 +2547,7 @@ class SetMetadataTool(Tool):
             job, base_dir='job_work', dir_only=True, obj_dir=True
         )
         for name, dataset in inp_data.items():
-            external_metadata = get_metadata_compute_strategy(app.config, job.id)
+            external_metadata = get_metadata_compute_strategy(app.config, job.id, tool_id=self.id)
             sa_session = app.model.context
             metadata_set_successfully = external_metadata.external_metadata_set_successfully(dataset, name, sa_session, working_directory=working_directory)
             if metadata_set_successfully:

lib/galaxy/tools/actions/metadata.py

@@ -85,7 +85,7 @@ class SetMetadataToolAction(ToolAction):
         job_working_dir = app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(job.id))
         datatypes_config = os.path.join(job_working_dir, 'registry.xml')
         app.datatypes_registry.to_xml_file(path=datatypes_config)
-        external_metadata_wrapper = get_metadata_compute_strategy(app.config, job.id)
+        external_metadata_wrapper = get_metadata_compute_strategy(app.config, job.id, tool_id=tool.id)
         output_datatasets_dict = {
             dataset_name: dataset,
         }

lib/galaxy/util/template.py

@@ -1,12 +1,12 @@
 """Entry point for the usage of Cheetah templating within Galaxy."""
 
-import sys
 import traceback
 from lib2to3.refactor import RefactoringTool
 
 import packaging.version
 from Cheetah.Compiler import Compiler
 from Cheetah.NameMapper import NotFound
+from Cheetah.Parser import ParseError
 from Cheetah.Template import Template
 from past.translation import myfixes
 
@@ -14,11 +14,8 @@ from . import unicodify
 
 # Skip libpasteurize fixers, which make sure code is py2 and py3 compatible.
 # This is not needed, we only translate code on py3.
-if sys.version_info.major > 2:
-    myfixes = [f for f in myfixes if not f.startswith('libpasteurize')]
-    refactoring_tool = RefactoringTool(myfixes, {'print_function': True})
-else:
-    myfixes = refactoring_tool = None
+myfixes = [f for f in myfixes if not f.startswith('libpasteurize')]
+refactoring_tool = RefactoringTool(myfixes, {'print_function': True})
 
 
 class FixedModuleCodeCompiler(Compiler):
@@ -60,14 +57,32 @@ def fill_template(template_text,
         context = kwargs
     if isinstance(python_template_version, str):
         python_template_version = packaging.version.parse(python_template_version)
-    klass = Template.compile(source=template_text, compilerClass=compiler_class)
+    try:
+        klass = Template.compile(source=template_text, compilerClass=compiler_class)
+    except ParseError as e:
+        # Might happen on invalid syntax within a cheetah statement, like `#if $smxsize <> 128.0`
+        if first_exception is None:
+            first_exception = e
+        if python_template_version.release[0] < 3 and retry > 0:
+            module_code = Template.compile(source=template_text, compilerClass=compiler_class, returnAClass=False).decode('utf-8')
+            module_code = futurize_preprocessor(module_code)
+            compiler_class = create_compiler_class(module_code)
+            return fill_template(
+                template_text=template_text,
+                context=context,
+                retry=retry - 1,
+                compiler_class=compiler_class,
+                first_exception=first_exception,
+                python_template_version=python_template_version,
+            )
+        raise first_exception or e
     t = klass(searchList=[context])
     try:
         return unicodify(t)
     except NotFound as e:
         if first_exception is None:
             first_exception = e
-        if refactoring_tool and python_template_version.release[0] < 3 and retry > 0:
+        if python_template_version.release[0] < 3 and retry > 0:
             tb = e.__traceback__
             last_stack = traceback.extract_tb(tb)[-1]
             if last_stack.name == '<listcomp>':
@@ -94,7 +109,7 @@ def fill_template(template_text,
     except Exception as e:
         if first_exception is None:
             first_exception = e
-        if sys.version_info.major > 2 and python_template_version.release[0] < 3 and not futurized:
+        if python_template_version.release[0] < 3 and not futurized:
             # Possibly an error caused by attempting to run python 2
             # template code on python 3. Run the generated module code
             # through futurize and hope for the best.

lib/galaxy/webapps/galaxy/api/pages.py

@@ -11,7 +11,11 @@ from galaxy.managers.pages import (
     PageSerializer
 )
 from galaxy.model.item_attrs import UsesAnnotations
-from galaxy.web import expose_api, expose_api_raw
+from galaxy.web import (
+    expose_api,
+    expose_api_anonymous_and_sessionless,
+    expose_api_raw_anonymous_and_sessionless
+)
 from galaxy.webapps.base.controller import (
     BaseAPIController,
     SharableItemSecurityMixin,
@@ -31,7 +35,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
         self.manager = PageManager(app)
         self.serializer = PageSerializer(app)
 
-    @expose_api
+    @expose_api_anonymous_and_sessionless
     def index(self, trans, deleted=False, **kwd):
         """
         index( self, trans, deleted=False, **kwd )
@@ -52,12 +56,14 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
             for row in r:
                 out.append(self.encode_all_ids(trans, row.to_dict(), True))
         else:
+            # Transaction user's pages (if any)
             user = trans.get_user()
             r = trans.sa_session.query(trans.app.model.Page).filter_by(user=user)
             if not deleted:
                 r = r.filter_by(deleted=False)
             for row in r:
                 out.append(self.encode_all_ids(trans, row.to_dict(), True))
+            # Published pages from other users
             r = trans.sa_session.query(trans.app.model.Page).filter(trans.app.model.Page.user != user).filter_by(published=True)
             if not deleted:
                 r = r.filter_by(deleted=False)
@@ -108,7 +114,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
         trans.sa_session.flush()
         return ''  # TODO: Figure out what to return on DELETE, document in guidelines!
 
-    @expose_api
+    @expose_api_anonymous_and_sessionless
     def show(self, trans, id, **kwd):
         """
         show( self, trans, id, **kwd )
@@ -127,7 +133,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
         self.manager.rewrite_content_for_export(trans, rval)
         return rval
 
-    @expose_api_raw
+    @expose_api_raw_anonymous_and_sessionless
     def show_pdf(self, trans, id, **kwd):
         """
         show( self, trans, id, **kwd )

lib/galaxy/webapps/galaxy/controllers/admin.py

@@ -589,8 +589,9 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
         status = kwd.get('status', 'done')
         for dtype in sorted(trans.app.datatypes_registry.datatype_elems,
                            key=lambda dtype: dtype.get('extension')):
-            datatypes.append(dtype.attrib)
-            keys |= set(dtype.attrib)
+            attrib = dict(dtype.attrib)
+            datatypes.append(attrib)
+            keys |= set(attrib.keys())
         return {'keys': list(keys), 'data': datatypes, 'message': message, 'status': status}
 
     @web.expose

lib/tool_shed/webapp/templates/base.mako

@@ -50,15 +50,15 @@
 ## Default javascripts
 <%def name="javascripts()">
     ## TODO: remove when all libs are required directly in modules
-    ${h.js(
-        'bundled/libs.chunk',
-        'bundled/base.chunk'
+    ${h.dist_js(
+        'libs.chunk',
+        'base.chunk'
     )}
     ${self.javascript_entry()}
 </%def>
 
 <%def name="javascript_entry()">
-    ${h.js('bundled/generic.bundled')}
+    ${h.dist_js('generic.bundled')}
 </%def>
 
 <%def name="javascript_app()">

lib/tool_shed/webapp/templates/base/base_panels.mako

@@ -29,16 +29,16 @@
 ## TODO: remove when all libs are required directly in modules
 <%def name="javascripts()">
     <!--- base/base_panels.mako javascripts() -->
-    ${h.js(
-        'bundled/libs.chunk',
-        'bundled/base.chunk'
+    ${h.dist_js(
+        'libs.chunk',
+        'base.chunk'
     )}
     ${ javascript_entry() }
 </%def>
 
 <%def name="javascript_entry()">
     <!-- base/base_panels.mako javascript_entry -->
-    ${ h.js('bundled/generic.bundled')}
+    ${ h.dist_js('generic.bundled')}
 </%def>
 
 <%def name="javascript_app()">

lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako

@@ -23,7 +23,6 @@
 
 <%def name="javascripts()">
     ${parent.javascripts()}
-    ## ${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )}
     ${common_javascripts(repository)}
     <script type="text/javascript">
     $( function() {

templates/admin/tool_shed_repository/browse_tool_dependency.mako

@@ -10,7 +10,6 @@
 
 <%def name="javascripts()">
     ${parent.javascripts()}
-    ## ${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.dynatree" )}
     ${browse_files(tool_dependency.name, tool_dependency.installation_directory( trans.app ))}
 </%def>
 

test/integration/objectstore/test_objectstore_datatype_upload.py

@@ -113,8 +113,7 @@ class BaseObjectstoreUploadTest(UploadTestDatatypeDataTestCase):
         temp_directory = cls._test_driver.mkdtemp()
         cls.object_stores_parent = temp_directory
         cls.object_store_config_path = os.path.join(temp_directory, "object_store_conf.xml")
-        # This doesn't quite work yet, fails with extra_files_path
-        # config["metadata_strategy"] = "extended"
+        config["metadata_strategy"] = "extended"
         config["outpus_to_working_dir"] = True
         config["retry_metadata_internally"] = False
         config["object_store_store_by"] = "uuid"

test/integration/objectstore/test_selection.py

@@ -44,6 +44,9 @@ class ObjectStoreSelectionIntegrationTestCase(BaseObjectStoreIntegrationTestCase
         cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config)
         config["job_config_file"] = JOB_CONFIG_FILE
         config["job_resource_params_file"] = JOB_RESOURCE_PARAMETERS_CONFIG_FILE
+        config["object_store_store_by"] = "uuid"
+        config["metadata_strategy"] = "extended"
+        config["outputs_to_working_directory"] = True
 
     def _object_store_counts(self):
         files_default_count = files_count(self.files_default_path)
@@ -61,6 +64,11 @@ class ObjectStoreSelectionIntegrationTestCase(BaseObjectStoreIntegrationTestCase
         assert dynamic_ebs == files_dynamic_ebs_count
         assert dynamic_s3 == files_dynamic_s3_count
 
+    def _assert_no_external_filename(self):
+        # Should maybe be its own test case ...
+        for external_filename_tuple in self._app.model.session.query(self._app.model.Dataset.external_filename).all():
+            assert external_filename_tuple[0] is None
+
     def test_tool_simple_constructs(self):
         with self.dataset_populator.test_history() as history_id:
 
@@ -103,3 +111,4 @@ class ObjectStoreSelectionIntegrationTestCase(BaseObjectStoreIntegrationTestCase
             }
             _run_tool("create_10", create_10_inputs)
             self._assert_file_counts(1, 2, 10, 10)
+            self._assert_no_external_filename()

test/integration/objectstore/test_swift_objectstore.py

@@ -89,8 +89,7 @@ class SwiftObjectStoreIntegrationTestCase(integration_util.IntegrationTestCase):
         cls.object_stores_parent = temp_directory
         config_path = os.path.join(temp_directory, "object_store_conf.xml")
         config["object_store_store_by"] = "uuid"
-        # This doesn't quite work yet, fails with extra_files_path
-        # config["metadata_strategy"] = "extended"
+        config["metadata_strategy"] = "extended"
         config["outpus_to_working_dir"] = True
         config["retry_metadata_internally"] = False
         with open(config_path, "w") as f:

test/unit/jobs/test_job_wrapper.py

@@ -163,6 +163,7 @@ class MockTool(object):
         self.tool_dir = "/path/to/tools"
         self.dependencies = []
         self.requires_galaxy_python_environment = False
+        self.id = 'mock_id'
 
     def build_dependency_shell_commands(self, job_directory):
         return TEST_DEPENDENCIES_COMMANDS

test/unit/util/test_fill_template.py

@@ -37,6 +37,12 @@ TWO_TO_THREE_TEMPLATE = """#set $a = [x for x in {'a': '1'}.iterkeys()][0]
 #set $c = [x for x in {'a': '1'}.itervalues()][0]
 $a $b $c"""
 
+INVALID_CHEETAH_SYNTAX = """#if 1 <> 1
+1 is not 1
+#else
+1 is 1
+#end if"""
+
 
 def test_fill_simple_template():
     template_str = str(fill_template(SIMPLE_TEMPLATE, {'a_list': [1, 2]}))
@@ -75,3 +81,8 @@ def test_gen_expr():
 def test_fix_template_two_to_three():
     template_str = fill_template(TWO_TO_THREE_TEMPLATE, python_template_version='2', retry=1)
     assert template_str == 'a a 1'
+
+
+def test_fix_template_invalid_cheetah():
+    template_str = fill_template(INVALID_CHEETAH_SYNTAX, python_template_version='2', retry=1)
+    assert template_str == "1 is 1\n"