Merge branch 'release_26.0' into dev

lib/galaxy/datatypes/data.py

@@ -510,7 +510,7 @@ class Data(metaclass=DataMeta):
     def _serve_binary_file_contents_as_text(self, trans, data, headers, file_size, max_peek_size):
         headers["content-type"] = "text/html"
         if file_size > max_peek_size:
-            headers["x-content-truncated"] = max_peek_size
+            headers["x-content-truncated"] = str(max_peek_size)
         with open(data.get_file_name(), "rb") as fh:
             return unicodify(fh.read(max_peek_size)), headers
 
@@ -524,7 +524,7 @@ class Data(metaclass=DataMeta):
         with compression_utils.get_fileobj(data.get_file_name(), "rb") as fh:
             # preview large text file
             headers["content-type"] = "text/html"
-            headers["x-content-truncated"] = max_peek_size
+            headers["x-content-truncated"] = str(max_peek_size)
             return unicodify(fh.read(max_peek_size)), headers
 
     def display_data(

lib/galaxy/job_execution/output_collect.py

@@ -15,9 +15,12 @@ from typing import (
 )
 
 from galaxy.model import (
+    Dataset,
     DatasetInstance,
     HistoryDatasetAssociation,
     HistoryDatasetCollectionAssociation,
+    JOB_IO_NAME_MAX_LENGTH,
+    JobOutputNameTooLongError,
 )
 from galaxy.model.dataset_collections import builder
 from galaxy.model.dataset_collections.structure import UninitializedTree
@@ -337,6 +340,11 @@ class SessionlessJobContext(SessionlessModelPersistenceContext, BaseJobContext):
             self.export_store.collection_datasets.add(collection_dataset.id)
 
     def add_output_dataset_association(self, name, dataset_instance):
+        if name and len(name) > JOB_IO_NAME_MAX_LENGTH:
+            raise JobOutputNameTooLongError(
+                f"Tool produced an output name that exceeds the {JOB_IO_NAME_MAX_LENGTH} character name length limit "
+                f"(got {len(name)} characters), tool is likely broken"
+            )
         assert self.export_store
         self.export_store.add_job_output_dataset_associations(self.get_job_id(), name, dataset_instance)
 
@@ -427,8 +435,14 @@ def collect_primary_datasets(job_context: BaseJobContext, output: dict[str, Data
                 storage_callbacks=storage_callbacks,
                 purged=outdata.dataset.purged,
             )
-            # Associate new dataset with job
-            job_context.add_output_dataset_association(f"__new_primary_file_{name}|{designation}__", primary_data)
+            try:
+                # Associate new dataset with job
+                job_context.add_output_dataset_association(f"__new_primary_file_{name}|{designation}__", primary_data)
+            except JobOutputNameTooLongError:
+                primary_data.dataset.state = Dataset.states.DISCARDED
+                primary_data.dataset.file_size = 0
+                job_context.add_datasets_to_history([primary_data], for_output_dataset=outdata)
+                raise
             job_context.add_datasets_to_history([primary_data], for_output_dataset=outdata)
             # Add dataset to return dict
             primary_datasets[name][designation] = primary_data

lib/galaxy/jobs/__init__.py

@@ -81,6 +81,7 @@ from galaxy.metadata import get_metadata_compute_strategy
 from galaxy.model import (
     Dataset,
     Job,
+    JobOutputNameTooLongError,
     store,
     Task,
 )
@@ -2143,10 +2144,10 @@ class MinimalJobWrapper(HasResourceParameters):
             except store.FileTracebackException as e:
                 job.traceback = e.traceback
                 log.exception(f"Problem generating command line for Job {job.id}.\n{job.traceback}")
-                raise
-            except Exception:
+                return fail(str(e), exception=e)
+            except Exception as e:
                 log.exception(f"problem importing job outputs. stdout [{job.stdout}] stderr [{job.stderr}]")
-                raise
+                return fail(str(e), exception=e)
         else:
             if self.tool.version_string_cmd:
                 version_filename = self.get_version_string_path()
@@ -2159,7 +2160,7 @@ class MinimalJobWrapper(HasResourceParameters):
             # importing metadata will discover outputs if extended metadata
             try:
                 self.discover_outputs(job, inp_data, out_data, out_collections, final_job_state=final_job_state)
-            except MaxDiscoveredFilesExceededError as e:
+            except (MaxDiscoveredFilesExceededError, JobOutputNameTooLongError) as e:
                 final_job_state = job.states.ERROR
                 job.job_messages = [
                     {

lib/galaxy/metadata/set_metadata.py

@@ -49,6 +49,7 @@ from galaxy.model import (
     DatasetInstance,
     HistoryDatasetAssociation,
     Job,
+    JobOutputNameTooLongError,
     store,
 )
 from galaxy.model.custom_types import total_size
@@ -346,7 +347,7 @@ def set_metadata_portable(
                 input_ext=input_ext,
             )
             collect_dynamic_outputs(job_context, output_collections)
-        except MaxDiscoveredFilesExceededError as e:
+        except (MaxDiscoveredFilesExceededError, JobOutputNameTooLongError) as e:
             final_job_state = Job.states.ERROR
             job_messages.append(
                 {
@@ -369,7 +370,6 @@ def set_metadata_portable(
                             continue
                         command_line_lines.append(line)
                     job.command_line = "".join(command_line_lines).strip()
-                    export_store.export_job(job, include_job_data=False)
 
     unnamed_id_to_path = {}
     unnamed_is_deferred = {}

lib/galaxy/model/__init__.py

@@ -1582,6 +1582,13 @@ class IoDicts(NamedTuple):
     out_collections: OutCollectionsDictT
 
 
+JOB_IO_NAME_MAX_LENGTH = 255
+
+
+class JobOutputNameTooLongError(ValueError):
+    pass
+
+
 class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
     """
     A job represents a request to run a tool given input datasets, tool
@@ -2022,6 +2029,14 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
     def add_parameter(self, name, value):
         self.parameters.append(JobParameter(name, value))
 
+    @staticmethod
+    def _check_name_length(name: str) -> None:
+        if name and len(name) > JOB_IO_NAME_MAX_LENGTH:
+            raise JobOutputNameTooLongError(
+                f"Tool produced an output name that exceeds the {JOB_IO_NAME_MAX_LENGTH} character name length limit "
+                f"(got {len(name)} characters), tool is likely broken"
+            )
+
     def add_input_dataset(self, name, dataset=None, dataset_id=None, adapter_json=None):
         assoc = JobToInputDatasetAssociation(name, dataset, adapter_json)
         if dataset is None and dataset_id is not None:
@@ -2030,6 +2045,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
         self.input_datasets.append(assoc)
 
     def add_output_dataset(self, name: str, dataset: "HistoryDatasetAssociation"):
+        self._check_name_length(name)
         joda = JobToOutputDatasetAssociation(name, dataset)
         if dataset.dataset.job is None:
             # Only set job if dataset doesn't already have associated job.
@@ -2049,11 +2065,13 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
         )
 
     def add_output_dataset_collection(self, name, dataset_collection_instance):
+        self._check_name_length(name)
         self.output_dataset_collection_instances.append(
             JobToOutputDatasetCollectionAssociation(name, dataset_collection_instance)
         )
 
     def add_implicit_output_dataset_collection(self, name, dataset_collection):
+        self._check_name_length(name)
         self.output_dataset_collections.append(
             JobToImplicitOutputDatasetCollectionAssociation(name, dataset_collection)
         )
@@ -2062,6 +2080,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
         self.input_library_datasets.append(JobToInputLibraryDatasetAssociation(name, dataset))
 
     def add_output_library_dataset(self, name, dataset):
+        self._check_name_length(name)
         self.output_library_datasets.append(JobToOutputLibraryDatasetAssociation(name, dataset))
 
     def add_post_job_action(self, pja):

lib/galaxy/model/store/discover.py

@@ -24,7 +24,11 @@ from typing import (
 import galaxy.model
 from galaxy import util
 from galaxy.exceptions import RequestParameterInvalidException
-from galaxy.model import LibraryFolder
+from galaxy.model import (
+    Dataset,
+    JobOutputNameTooLongError,
+    LibraryFolder,
+)
 from galaxy.model.dataset_collections.builder import BoundCollectionBuilder
 from galaxy.model.tags import GalaxySessionlessTagHandler
 from galaxy.objectstore import (
@@ -435,20 +439,27 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
             element_datasets["rows"].append(discovered_file.match.row)
 
         self.add_tags_to_datasets(datasets=element_datasets["datasets"], tag_lists=element_datasets["tag_lists"])
-        for element_identifiers, dataset, row in zip(
-            element_datasets["element_identifiers"], element_datasets["datasets"], element_datasets["rows"]
-        ):
-            current_builder: CollectionBuilder = root_collection_builder
-            for element_identifier in element_identifiers[:-1]:
-                current_builder = current_builder.get_level(element_identifier, row=row)
-                if row:
-                    row = None
-            current_builder.add_dataset(element_identifiers[-1], dataset, row=row)
+        try:
+            for element_identifiers, dataset, row in zip(
+                element_datasets["element_identifiers"], element_datasets["datasets"], element_datasets["rows"]
+            ):
+                current_builder: CollectionBuilder = root_collection_builder
+                for element_identifier in element_identifiers[:-1]:
+                    current_builder = current_builder.get_level(element_identifier, row=row)
+                    if row:
+                        row = None
+                current_builder.add_dataset(element_identifiers[-1], dataset, row=row)
 
-            # Associate new dataset with job
-            element_identifier_str = ":".join(element_identifiers)
-            association_name = f"__new_primary_file_{name}|{element_identifier_str}__"
-            self.add_output_dataset_association(association_name, dataset)
+                # Associate new dataset with job
+                element_identifier_str = ":".join(element_identifiers)
+                association_name = f"__new_primary_file_{name}|{element_identifier_str}__"
+                self.add_output_dataset_association(association_name, dataset)
+        except JobOutputNameTooLongError:
+            for dataset in element_datasets["datasets"]:
+                dataset.dataset.state = Dataset.states.DISCARDED
+                dataset.dataset.file_size = 0
+            self.add_datasets_to_history(element_datasets["datasets"])
+            raise
 
         add_datasets_timer = ExecutionTimer()
         self.add_datasets_to_history(element_datasets["datasets"])

lib/galaxy/tools/__init__.py

@@ -962,6 +962,7 @@ class JobContext(BaseJobContext):
         return hdca
 
     def add_output_dataset_association(self, name, dataset):
+        Job._check_name_length(name)
         assoc = JobToOutputDatasetAssociation(name, dataset)
         assoc.job = self.job
         self.sa_session.add(assoc)

lib/galaxy/webapps/galaxy/controllers/dataset.py

@@ -82,8 +82,10 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
         return "This link may not be followed from within Galaxy."
 
     @web.expose_api_raw_anonymous_and_sessionless
-    def get_metadata_file(self, trans, hda_id, metadata_name, **kwd):
+    def get_metadata_file(self, trans, hda_id=None, metadata_name=None, **kwd):
         """Allows the downloading of metadata files associated with datasets (eg. bai index for bam files)"""
+        if hda_id is None or metadata_name is None:
+            raise RequestParameterInvalidException("Required parameters 'hda_id' and 'metadata_name' are missing.")
         # Backward compatibility with legacy links, should use `/api/datasets/{hda_id}/get_metadata_file` instead
         fh, headers = self.service.get_metadata_file(
             trans, history_content_id=self.decode_id(hda_id), metadata_file=metadata_name, open_file=True
@@ -597,7 +599,9 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
                             )
                         )
                     else:
-                        raise Exception(f"Attempted a view action ({app_action}) on a non-ready display application")
+                        return trans.show_error_message(
+                            f"Attempted a view action ({app_action}) on a non-ready display application"
+                        )
             return dict(msg=msg)
         return trans.show_error_message(
             "You do not have permission to view this dataset at an external display application."

lib/galaxy/workflow/extract.py

@@ -344,18 +344,20 @@ class WorkflowSummary:
             for cja_assoc in cja:
                 cja_job = cja_assoc.job
                 self.job_id2representative_job[cja_job.id] = representative_job
-        # This whole elif condition may no longer be needed do to additional
-        # tracking with creating_job_associations. Will delete at some point.
+        # Fallback for implicit output collections lacking creating_job_associations
+        # (e.g. reached via Sentry GALAXY-MAIN-121W / issue #22359). Trace via a leaf
+        # HDA's creating job instead.
         elif dataset_collection.implicit_output_name:
             # TODO: Optimize db call
             element = dataset_collection.collection.first_dataset_element
-            if not element:
-                # Got no dataset instance to walk back up to creating job.
+            dataset_instance = element.hda if element else None
+            if not dataset_instance:
+                # Got no dataset instance to walk back up to creating job
+                # (empty collection, or leaf element is not an HDA - e.g. LDDA).
                 # TODO track this via tool request model
                 self.jobs[DatasetCollectionCreationJob(dataset_collection)] = [(None, dataset_collection)]
                 return
-            dataset_instance = element.hda
-            if dataset_instance is None or not self.__check_state(dataset_instance):
+            if not self.__check_state(dataset_instance):
                 # Just checking the state of one instance, don't need more but
                 # makes me wonder if even need this check at all?
                 return

lib/galaxy_test/api/test_tools.py

@@ -2437,6 +2437,21 @@ class TestToolsApi(ApiTestCase, TestsTools):
         hdca = self._get(f"histories/{history_id}/contents/dataset_collections/{implicit_collections[0]['id']}").json()
         assert hdca["elements"][0]["object"]["elements"][0]["object"]["elements"][0]["element_identifier"] == "forward"
 
+    @skip_without_tool("discover_long_name")
+    def test_long_output_name_fails_gracefully(self, history_id):
+        # Short name succeeds
+        response = self._run("discover_long_name", history_id, {"output_name": "normal_name"})
+        self._assert_status_code_is(response, 200)
+        self.dataset_populator.wait_for_job(response.json()["jobs"][0]["id"], assert_ok=True)
+        # Long name fails with clear error
+        response = self._run("discover_long_name", history_id, {"output_name": "a" * 240})
+        self._assert_status_code_is(response, 200)
+        job_id = response.json()["jobs"][0]["id"]
+        self.dataset_populator.wait_for_job(job_id, assert_ok=False)
+        job_details = self.dataset_populator.get_job_details(job_id, full=True).json()
+        assert job_details["state"] == "error"
+        assert "255 character" in job_details["job_messages"][0]["desc"]
+
     def _bed_list(self, history_id):
         bed1_contents = open(self.get_filename("1.bed")).read()
         bed2_contents = open(self.get_filename("2.bed")).read()

test/functional/tools/discover_long_name.xml

@@ -0,0 +1,13 @@
+<tool id="discover_long_name" name="discover_long_name" version="0.1.0" profile="20.09">
+    <command><![CDATA[
+echo "data" > '${output_name}.txt'
+    ]]></command>
+    <inputs>
+        <param name="output_name" type="text" label="Output name" />
+    </inputs>
+    <outputs>
+        <data format="txt" name="output">
+            <discover_datasets pattern="__name_and_ext__" visible="true" />
+        </data>
+    </outputs>
+</tool>

test/functional/tools/sample_tool_conf.xml

@@ -221,6 +221,7 @@
   <tool file="collection_paired_or_unpaired.xml" />
   <tool file="collection_list_paired_or_unpaired.xml" />
   <tool file="discover_sort_by.xml" />
+  <tool file="discover_long_name.xml" />
   <tool file="discover_default_ext.xml" />
   <tool file="expression_forty_two.xml" />
   <tool file="expression_pick_larger_file.xml" />