diff --git a/config/plugins/interactive_environments/common/templates/ie.mako b/config/plugins/interactive_environments/common/templates/ie.mako index 98fe7919c46..ba4c774018b 100644 --- a/config/plugins/interactive_environments/common/templates/ie.mako +++ b/config/plugins/interactive_environments/common/templates/ie.mako @@ -1,8 +1,8 @@ <%def name="load_default_js()"> ${h.css( 'base' ) } -${h.js('bundled/libs.chunk', - 'bundled/base.chunk', - 'bundled/generic.bundled')} +${h.dist_js('libs.chunk', + 'base.chunk', + 'generic.bundled')} <%def name="default_javascript_variables()"> diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 5783b0b3b29..0cf7d5356c3 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -918,7 +918,7 @@ class JobWrapper(HasResourceParameters): metadata_strategy_override = None if job.tasks: metadata_strategy_override = "directory" - self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override) + self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override, tool_id=job.tool_id) self.__commands_in_new_shell = True self.__user_system_pwent = None diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py index aa041eb1df6..390a0ac780b 100644 --- a/lib/galaxy/metadata/__init__.py +++ b/lib/galaxy/metadata/__init__.py @@ -21,11 +21,11 @@ log = getLogger(__name__) SET_METADATA_SCRIPT = 'from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()' -def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None): +def get_metadata_compute_strategy(config, job_id, metadata_strategy_override=None, tool_id=None): metadata_strategy = metadata_strategy_override or config.metadata_strategy if metadata_strategy == "legacy": return JobExternalOutputMetadataWrapper(job_id) - elif metadata_strategy == "extended": + elif metadata_strategy == "extended" and tool_id != "__SET_METADATA__": return ExtendedDirectoryMetadataGenerator(job_id) else: return PortableDirectoryMetadataGenerator(job_id) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 6ca7bd8abe9..865a247c4f4 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -168,7 +168,6 @@ def set_metadata_portable(): if os.path.exists(COMMAND_VERSION_FILENAME): version_string = open(COMMAND_VERSION_FILENAME).read() - # TODO: handle outputs_to_working_directory? from galaxy.util.expressions import ExpressionContext job_context = ExpressionContext(dict(stdout=tool_stdout, stderr=tool_stderr)) @@ -237,7 +236,10 @@ def set_metadata_portable(): dataset.set_size() if 'uuid' in context: dataset.dataset.uuid = context['uuid'] - object_store.update_from_file(dataset.dataset, create=True) + if dataset_filename_override and dataset_filename_override != dataset.file_name: + # This has to be a job with outputs_to_working_directory set. + # We update the object store with the created output file. + object_store.update_from_file(dataset.dataset, file_name=dataset_filename_override, create=True) from galaxy.job_execution.output_collect import collect_extra_files collect_extra_files(object_store, dataset, ".") if galaxy.model.Job.states.ERROR == final_job_state: @@ -265,11 +267,9 @@ def set_metadata_portable(): if context_key in context: context_value = context[context_key] setattr(dataset, context_key, context_value) - - if extended_metadata_collection: - export_store.add_dataset(dataset) - else: - cPickle.dump(dataset, open(filename_out, 'wb+')) + # We never want to persist the external_filename. + dataset.dataset.external_filename = None + export_store.add_dataset(dataset) else: dataset.metadata.to_JSON_dict(filename_out) # write out results of set_meta diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 3e2a6f43d0c..1c3c0d69435 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -368,7 +368,7 @@ class DiskObjectStore(ConcreteObjectStore): :param extra_dirs: Keys are string, values are directory paths. """ super().__init__(config, config_dict) - self.file_path = config_dict.get("files_dir") or config.file_path + self.file_path = os.path.abspath(config_dict.get("files_dir") or config.file_path) @classmethod def parse_xml(clazz, config_xml): diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 20814e6703d..31465789884 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2547,7 +2547,7 @@ class SetMetadataTool(Tool): job, base_dir='job_work', dir_only=True, obj_dir=True ) for name, dataset in inp_data.items(): - external_metadata = get_metadata_compute_strategy(app.config, job.id) + external_metadata = get_metadata_compute_strategy(app.config, job.id, tool_id=self.id) sa_session = app.model.context metadata_set_successfully = external_metadata.external_metadata_set_successfully(dataset, name, sa_session, working_directory=working_directory) if metadata_set_successfully: diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index 5667b6b770f..325d3825fef 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -85,7 +85,7 @@ class SetMetadataToolAction(ToolAction): job_working_dir = app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(job.id)) datatypes_config = os.path.join(job_working_dir, 'registry.xml') app.datatypes_registry.to_xml_file(path=datatypes_config) - external_metadata_wrapper = get_metadata_compute_strategy(app.config, job.id) + external_metadata_wrapper = get_metadata_compute_strategy(app.config, job.id, tool_id=tool.id) output_datatasets_dict = { dataset_name: dataset, } diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index bd3fa9946b9..890b4b2d5c1 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -1,12 +1,12 @@ """Entry point for the usage of Cheetah templating within Galaxy.""" -import sys import traceback from lib2to3.refactor import RefactoringTool import packaging.version from Cheetah.Compiler import Compiler from Cheetah.NameMapper import NotFound +from Cheetah.Parser import ParseError from Cheetah.Template import Template from past.translation import myfixes @@ -14,11 +14,8 @@ from . import unicodify # Skip libpasteurize fixers, which make sure code is py2 and py3 compatible. # This is not needed, we only translate code on py3. -if sys.version_info.major > 2: - myfixes = [f for f in myfixes if not f.startswith('libpasteurize')] - refactoring_tool = RefactoringTool(myfixes, {'print_function': True}) -else: - myfixes = refactoring_tool = None +myfixes = [f for f in myfixes if not f.startswith('libpasteurize')] +refactoring_tool = RefactoringTool(myfixes, {'print_function': True}) class FixedModuleCodeCompiler(Compiler): @@ -60,14 +57,32 @@ def fill_template(template_text, context = kwargs if isinstance(python_template_version, str): python_template_version = packaging.version.parse(python_template_version) - klass = Template.compile(source=template_text, compilerClass=compiler_class) + try: + klass = Template.compile(source=template_text, compilerClass=compiler_class) + except ParseError as e: + # Might happen on invalid syntax within a cheetah statement, like `#if $smxsize <> 128.0` + if first_exception is None: + first_exception = e + if python_template_version.release[0] < 3 and retry > 0: + module_code = Template.compile(source=template_text, compilerClass=compiler_class, returnAClass=False).decode('utf-8') + module_code = futurize_preprocessor(module_code) + compiler_class = create_compiler_class(module_code) + return fill_template( + template_text=template_text, + context=context, + retry=retry - 1, + compiler_class=compiler_class, + first_exception=first_exception, + python_template_version=python_template_version, + ) + raise first_exception or e t = klass(searchList=[context]) try: return unicodify(t) except NotFound as e: if first_exception is None: first_exception = e - if refactoring_tool and python_template_version.release[0] < 3 and retry > 0: + if python_template_version.release[0] < 3 and retry > 0: tb = e.__traceback__ last_stack = traceback.extract_tb(tb)[-1] if last_stack.name == '': @@ -94,7 +109,7 @@ def fill_template(template_text, except Exception as e: if first_exception is None: first_exception = e - if sys.version_info.major > 2 and python_template_version.release[0] < 3 and not futurized: + if python_template_version.release[0] < 3 and not futurized: # Possibly an error caused by attempting to run python 2 # template code on python 3. Run the generated module code # through futurize and hope for the best. diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py index 2c0d3a4a5ff..8cf165886d3 100644 --- a/lib/galaxy/webapps/galaxy/api/pages.py +++ b/lib/galaxy/webapps/galaxy/api/pages.py @@ -11,7 +11,11 @@ from galaxy.managers.pages import ( PageSerializer ) from galaxy.model.item_attrs import UsesAnnotations -from galaxy.web import expose_api, expose_api_raw +from galaxy.web import ( + expose_api, + expose_api_anonymous_and_sessionless, + expose_api_raw_anonymous_and_sessionless +) from galaxy.webapps.base.controller import ( BaseAPIController, SharableItemSecurityMixin, @@ -31,7 +35,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati self.manager = PageManager(app) self.serializer = PageSerializer(app) - @expose_api + @expose_api_anonymous_and_sessionless def index(self, trans, deleted=False, **kwd): """ index( self, trans, deleted=False, **kwd ) @@ -52,12 +56,14 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati for row in r: out.append(self.encode_all_ids(trans, row.to_dict(), True)) else: + # Transaction user's pages (if any) user = trans.get_user() r = trans.sa_session.query(trans.app.model.Page).filter_by(user=user) if not deleted: r = r.filter_by(deleted=False) for row in r: out.append(self.encode_all_ids(trans, row.to_dict(), True)) + # Published pages from other users r = trans.sa_session.query(trans.app.model.Page).filter(trans.app.model.Page.user != user).filter_by(published=True) if not deleted: r = r.filter_by(deleted=False) @@ -108,7 +114,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati trans.sa_session.flush() return '' # TODO: Figure out what to return on DELETE, document in guidelines! - @expose_api + @expose_api_anonymous_and_sessionless def show(self, trans, id, **kwd): """ show( self, trans, id, **kwd ) @@ -127,7 +133,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati self.manager.rewrite_content_for_export(trans, rval) return rval - @expose_api_raw + @expose_api_raw_anonymous_and_sessionless def show_pdf(self, trans, id, **kwd): """ show( self, trans, id, **kwd ) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index c1431f31d9c..470b6a7241d 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -589,8 +589,9 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP status = kwd.get('status', 'done') for dtype in sorted(trans.app.datatypes_registry.datatype_elems, key=lambda dtype: dtype.get('extension')): - datatypes.append(dtype.attrib) - keys |= set(dtype.attrib) + attrib = dict(dtype.attrib) + datatypes.append(attrib) + keys |= set(attrib.keys()) return {'keys': list(keys), 'data': datatypes, 'message': message, 'status': status} @web.expose diff --git a/lib/tool_shed/webapp/templates/base.mako b/lib/tool_shed/webapp/templates/base.mako index 8c6e3f2ba1d..c72f5687ddf 100644 --- a/lib/tool_shed/webapp/templates/base.mako +++ b/lib/tool_shed/webapp/templates/base.mako @@ -50,15 +50,15 @@ ## Default javascripts <%def name="javascripts()"> ## TODO: remove when all libs are required directly in modules - ${h.js( - 'bundled/libs.chunk', - 'bundled/base.chunk' + ${h.dist_js( + 'libs.chunk', + 'base.chunk' )} ${self.javascript_entry()} <%def name="javascript_entry()"> - ${h.js('bundled/generic.bundled')} + ${h.dist_js('generic.bundled')} <%def name="javascript_app()"> diff --git a/lib/tool_shed/webapp/templates/base/base_panels.mako b/lib/tool_shed/webapp/templates/base/base_panels.mako index 4a231563b5f..1e1f81ac5d6 100644 --- a/lib/tool_shed/webapp/templates/base/base_panels.mako +++ b/lib/tool_shed/webapp/templates/base/base_panels.mako @@ -29,16 +29,16 @@ ## TODO: remove when all libs are required directly in modules <%def name="javascripts()"> - ${h.js( - 'bundled/libs.chunk', - 'bundled/base.chunk' + ${h.dist_js( + 'libs.chunk', + 'base.chunk' )} ${ javascript_entry() } <%def name="javascript_entry()"> - ${ h.js('bundled/generic.bundled')} + ${ h.dist_js('generic.bundled')} <%def name="javascript_app()"> diff --git a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako index e2fc5dd4306..634f957817e 100644 --- a/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako +++ b/lib/tool_shed/webapp/templates/webapps/tool_shed/repository/upload.mako @@ -23,7 +23,6 @@ <%def name="javascripts()"> ${parent.javascripts()} - ## ${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} ${common_javascripts(repository)}