diff --git a/client/src/components/Tool/ToolHelpForum.vue b/client/src/components/Tool/ToolHelpForum.vue index 912b417a425..49589dec860 100644 --- a/client/src/components/Tool/ToolHelpForum.vue +++ b/client/src/components/Tool/ToolHelpForum.vue @@ -6,10 +6,12 @@ import { computed, onMounted, ref } from "vue"; import { GalaxyApi } from "@/api"; import { galaxyLogo } from "@/components/icons/galaxyIcons"; import { useConfigStore } from "@/stores/configurationStore"; +import { errorMessageAsString } from "@/utils/simple-error"; import { getShortToolId } from "@/utils/tool"; import { createTopicUrl, type HelpForumPost, type HelpForumTopic, useHelpURLs } from "./helpForumUrls"; +import Alert from "@/components/Alert.vue"; import Heading from "@/components/Common/Heading.vue"; import ExternalLink from "@/components/ExternalLink.vue"; @@ -22,6 +24,7 @@ const toolHelpTag = "tool-help"; const topics = ref([]); const posts = ref([]); +const errorMessage = ref(""); const helpAvailable = computed(() => topics.value.length > 0); const root = ref(null); @@ -36,7 +39,7 @@ onMounted(async () => { }, }); if (error) { - console.error("Error fetching help forum data", error); + errorMessage.value = errorMessageAsString(error, "Failed to search the Help Forum."); } topics.value = data?.topics ?? []; @@ -66,12 +69,14 @@ const configStore = useConfigStore();
Help Forum + +

Following questions on the Help Forum may be related to this tool:

-

+

There are no questions on the Help Forum about this tool. diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue index abb7a462fb6..192a465b4e0 100644 --- a/client/src/components/Workflow/Editor/Index.vue +++ b/client/src/components/Workflow/Editor/Index.vue @@ -1252,7 +1252,17 @@ export default { const stepData = action.getNewStepData(); const response = await getModule( - { name, type, content_id: contentId, tool_state: state, tool_uuid: toolUuid }, + { + name, + type, + content_id: contentId, + tool_state: state, + tool_uuid: toolUuid, + // Request the latest version, mirroring what `routeToTool` does with `&version=latest`. + // Without this, toolshed tools whose GUID includes the version would resolve to + // that specific (possibly old) version rather than the latest. + tool_version: type === "tool" && !toolUuid ? "latest" : undefined, + }, stepData.id, this.stateStore.setLoadingState, ); @@ -1264,6 +1274,10 @@ export default { inputs: response.inputs, outputs: response.outputs, config_form: response.config_form, + // Use the resolved content_id and tool_version from the response which + // references the latest tool version, rather than what comes from the GUID + content_id: response.content_id || stepData.content_id, + tool_version: response.tool_version || stepData.tool_version, }; this.stepStore.updateStep(updatedStep); diff --git a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue index 06a2aecd549..c9fb1774b06 100644 --- a/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue +++ b/client/src/components/Workflow/Run/WorkflowRunFormSimple.vue @@ -195,9 +195,18 @@ function buildFormInputs() { if (stepType === "data_input" || stepType === "data_collection_input") { // Note: This is different from workflow landings because `WorkflowInvocationRequestModel` // does not provide an object with `values` property. - stepAsInput.value = { - values: !Array.isArray(value) ? [value] : value, - }; + // Optional data inputs left empty on the original run come back as `null` (or + // arrays containing `null`). Filter those out so we don't poison FormData with + // `{values: [null]}`, which crashes its `onMounted` hook on `"src" in null` and + // leaves the bad wrapper in formData to be sent to the server. + const valuesArray = (Array.isArray(value) ? value : [value]).filter( + (v) => v !== null && v !== undefined, + ); + if (valuesArray.length > 0) { + stepAsInput.value = { + values: valuesArray, + }; + } } else { stepAsInput.value = value; } diff --git a/client/src/entry/analysis/modules/Home.vue b/client/src/entry/analysis/modules/Home.vue index ca2bfff931f..1446d62ef6c 100644 --- a/client/src/entry/analysis/modules/Home.vue +++ b/client/src/entry/analysis/modules/Home.vue @@ -10,6 +10,8 @@ diff --git a/client/visualizations.yml b/client/visualizations.yml index d687def32e6..41a072c891e 100644 --- a/client/visualizations.yml +++ b/client/visualizations.yml @@ -122,7 +122,7 @@ venn: version: 0.0.8 vintent: package: "@galaxyproject/vintent" - version: 0.0.0 + version: 0.0.4 vitessce: package: "@galaxyproject/vitessce" version: 0.0.5 diff --git a/lib/galaxy/app/__init__.py b/lib/galaxy/app/__init__.py index be1f9c07f72..d7d01562095 100644 --- a/lib/galaxy/app/__init__.py +++ b/lib/galaxy/app/__init__.py @@ -878,7 +878,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT self._register_singleton(AgentRegistry, agent_registry) self._register_singleton( AgentService, - AgentService(self.config, JobQueryManager(self), agent_registry), + AgentService(self.config, JobQueryManager(self, self.history_manager), agent_registry), ) self.dependency_resolvers_view = self._register_singleton( diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 03771c9a233..de8724367f4 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -575,6 +575,7 @@ + diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index 24117b1227e..e2bedca767c 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -24,6 +24,10 @@ log = logging.getLogger(__name__) OWL_MARKER = re.compile(r"\]*>\s*)?(?:\s*)*<([A-Za-z_][\w.-]*:)?TEI(?:\s|>|/)", + re.DOTALL, +) @dataproviders.decorators.has_dataproviders @@ -113,6 +117,27 @@ class CisML(GenericXml): dataset.blurb = "file purged from disk" +class Tei(GenericXml): + """Text Encoding Initiative XML data.""" + + file_ext = "tei" + + def set_peek(self, dataset: DatasetProtocol, **kwd) -> None: + """Set the peek and blurb text""" + if not dataset.dataset.purged: + dataset.peek = data.get_file_peek(dataset.get_file_name()) + dataset.blurb = "TEI XML data" + else: + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" + + def sniff_prefix(self, file_prefix: FilePrefix) -> bool: + """ + Determines whether the file is TEI XML. + """ + return bool(file_prefix.search(TEI_MARKER)) + + class Dzi(GenericXml): """ Deep zoom image format, see diff --git a/lib/galaxy/files/sources/invenio.py b/lib/galaxy/files/sources/invenio.py index f5660b3e8c0..3e4d895ba95 100644 --- a/lib/galaxy/files/sources/invenio.py +++ b/lib/galaxy/files/sources/invenio.py @@ -269,10 +269,12 @@ class InvenioRepositoryInteractor(RDMRepositoryInteractor): """Gets the records in the repository and returns the total count of records.""" params: dict[str, Any] = {} request_url = self.records_url + if self.plugin.get_authorization_token(context) or write_intent: + # Authenticated users should browse only their own records. + request_url = self.user_records_url if write_intent: # Only draft records owned by the user can be written to. params["is_published"] = "false" - request_url = self.user_records_url size, page = self._to_size_page(limit, offset) params["size"] = size params["page"] = page @@ -297,7 +299,7 @@ class InvenioRepositoryInteractor(RDMRepositoryInteractor): writeable: bool, query: Optional[str] = None, ) -> list[RemoteFile]: - conditionally_draft = "/draft" if writeable else "" + conditionally_draft = "/draft" if writeable or self._is_draft_record(container_id, context) else "" request_url = f"{self.records_url}/{container_id}{conditionally_draft}/files" response_data = self._get_response(context, request_url) return self._get_record_files_from_response(container_id, response_data) diff --git a/lib/galaxy/files/templates/examples/production_zenodo.yml b/lib/galaxy/files/templates/examples/production_zenodo.yml index 0623ff27ff0..46a6395913b 100644 --- a/lib/galaxy/files/templates/examples/production_zenodo.yml +++ b/lib/galaxy/files/templates/examples/production_zenodo.yml @@ -32,7 +32,9 @@ The personal access token to use to connect to Zenodo. Go to your Account Settings and then go to Applications here https://zenodo.org/account/settings/applications/. You can generate a new `Personal Access Token` if you don't have one yet. This will allow Galaxy to display your draft records and upload files to them. If you enabled the option to export data - from Galaxy to Zenodo, make sure to enable the `deposit:write` scope when creating the token. + from Galaxy to Zenodo, make sure to enable the `deposit:write` scope when creating the token. + + **Note**: If you provide a token, you will be able to browse **only your own records**, if you don't provide a token, you will be able to browse all public records in Zenodo. configuration: type: zenodo diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index d997d878b66..04ed5e2cd0a 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -190,9 +190,10 @@ def safe_aliased(model_class: type[T], name: str) -> type[T]: class JobManager: - def __init__(self, app: StructuredApp): + def __init__(self, app: StructuredApp, history_manager: HistoryManager): self.app = app self.dataset_manager = DatasetManager(app) + self.history_manager = history_manager def index_query(self, trans: ProvidesUserContext, payload: JobIndexQueryPayload) -> sqlalchemy.engine.ScalarResult: """The caller is responsible for security checks on the resulting job if @@ -364,15 +365,26 @@ class JobManager: elif trans.galaxy_session: belongs_to_user = job.session_id == trans.galaxy_session.id if not trans.user_is_admin and not belongs_to_user: - # Check access granted via output datasets. - if not job.output_datasets: - raise ItemAccessibilityException("Job has no output datasets.") - for data_assoc in job.output_datasets: - if not self.dataset_manager.is_accessible(data_assoc.dataset.dataset, trans.user): - raise ItemAccessibilityException("You are not allowed to rerun this job.") + if not self._user_can_access_job(job, trans.user): + raise ItemAccessibilityException("You are not allowed to access this job.") trans.sa_session.refresh(job) return job + def _user_can_access_job(self, job: Job, user: Optional[User]) -> bool: + has_outputs = bool(job.output_datasets) or bool(job.output_dataset_collection_instances) + if has_outputs: + datasets_ok = all( + self.dataset_manager.is_accessible(da.dataset.dataset, user) for da in job.output_datasets + ) + collections_ok = all( + self.dataset_manager.is_accessible(hda.dataset, user) + for hdca_assoc in job.output_dataset_collection_instances + for hda in hdca_assoc.dataset_collection_instance.dataset_instances + ) + if datasets_ok and collections_ok: + return True + return job.history is not None and self.history_manager.is_accessible(job.history, user) + def get_job_console_output( self, trans, job, stdout_position=-1, stdout_length=0, stderr_position=-1, stderr_length=0 ): diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 0ac8934c68c..fe9f7ffd176 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -143,7 +143,7 @@ class GalaxyInternalMarkdownDirectiveHandler(metaclass=abc.ABCMeta): hda_manager = trans.app.hda_manager history_manager = trans.app.history_manager workflow_manager = trans.app.workflow_manager - job_manager = JobManager(trans.app) + job_manager = JobManager(trans.app, history_manager) collection_manager = trans.app.dataset_collection_manager def _remap(container, line): diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index b0ddd6c1649..2ef2777ada3 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -24,7 +24,10 @@ from boltons.iterutils import remap from packaging.version import Version from galaxy import model -from galaxy.exceptions import ToolInputsNotOKException +from galaxy.exceptions import ( + ToolInputsNotOKException, + ToolInputsNotReadyException, +) from galaxy.model import ( PostJobAction, ToolRequest, @@ -47,6 +50,7 @@ from galaxy.tools.execution_helpers import ( ToolExecutionCache, ) from galaxy.tools.parameters.workflow_utils import is_runtime_value +from galaxy.work.context import WorkRequestContext from ._types import ( ToolRequestT, ToolStateJobInstancePopulatedT, @@ -93,6 +97,71 @@ class MappingParameters(NamedTuple): assert self.validated_param_template is not None assert self.validated_param_combinations is not None + def example_params(self, trans: WorkRequestContext) -> ToolStateJobInstancePopulatedT: + """Representative per-job params for output-structure determination. + + Normally returns ``param_combinations[0]``. When the request + produces zero jobs (e.g. mapping over an empty collection), + falls back to a resolved copy of ``param_template``: batch + wrappers and raw ``{"src", "id"}`` refs are replaced with + HDCA/DCE ORM objects. Raises :class:`ToolInputsNotReadyException` + if a referenced collection exists but is not populated yet, so + the scheduler retries instead of surfacing the cryptic + "Referenced input parameter is not a collection." error. + """ + if self.param_combinations: + return self.param_combinations[0] + return _resolve_template(self.param_template, trans) + + +def _resolve_template(template: ToolRequestT, trans: WorkRequestContext) -> ToolStateJobInstancePopulatedT: + return {key: _resolve_template_value(value, trans) for key, value in template.items()} + + +def _resolve_template_value(value: Any, trans: WorkRequestContext) -> Any: + if isinstance(value, dict): + values = value.get("values") + if ( + isinstance(values, list) + and values + and isinstance(values[0], dict) + and "src" in values[0] + and "id" in values[0] + ): + return _resolve_collection_ref(values[0], trans, raw_fallback=value) + if "src" in value and "id" in value: + return _resolve_collection_ref(value, trans, raw_fallback=value) + return {k: _resolve_template_value(v, trans) for k, v in value.items()} + if isinstance(value, list): + return [_resolve_template_value(v, trans) for v in value] + return value + + +def _resolve_collection_ref( + ref: dict[str, Any], + trans: WorkRequestContext, + raw_fallback: Any, +) -> Union[model.HistoryDatasetCollectionAssociation, model.DatasetCollectionElement, Any]: + src = ref.get("src") + rid = ref.get("id") + if rid is None or src not in ("hdca", "dce"): + return raw_fallback + decoded = rid if isinstance(rid, int) else trans.security.decode_id(rid) + sa_session = trans.sa_session + if src == "hdca": + hdca = sa_session.get(model.HistoryDatasetCollectionAssociation, decoded) + if hdca is None: + return raw_fallback + if not hdca.collection.populated_optimized: + raise ToolInputsNotReadyException("An input collection is not populated.") + return hdca + dce = sa_session.get(model.DatasetCollectionElement, decoded) + if dce is None or dce.child_collection is None: + return raw_fallback + if not dce.child_collection.populated_optimized: + raise ToolInputsNotReadyException("An input collection is not populated.") + return dce + def execute_async( trans, @@ -434,13 +503,7 @@ class ExecutionTracker: @property def example_params(self): - if self.mapping_params.param_combinations: - return self.mapping_params.param_combinations[0] - else: - # TODO: This isn't quite right - what we want is something like param_template wrapped, - # need a test case with an output filter applied to an empty list, still this is - # an improvement over not allowing mapping of empty lists. - return self.mapping_params.param_template + return self.mapping_params.example_params(self.trans) @property def job_count(self): @@ -541,7 +604,7 @@ class ExecutionTracker: collection_type_description = ( self.trans.app.dataset_collection_manager.collection_type_descriptions.for_collection_type( - input_collection.collection.collection_type + get_collection(input_collection).collection_type ) ) subcollection_mapping_type = None diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index cbbdea95a5a..f59c3f1ff7a 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2134,6 +2134,25 @@ ItemFromSrcCollection = Union[ ] +def _decode_dataset_id(value, security: "IdEncodingHelper", parameter_name: str) -> int: + """Coerce a value into an integer dataset PK or raise ParameterValueError. + + Accepts int, digit string, or 16-char encoded id. Anything else + (including ``src:...``-prefixed strings, which should arrive as + ``{src, id}`` dicts via :func:`src_id_to_item`) is rejected so + malformed input surfaces as a 4xx instead of a SQL crash. + """ + if isinstance(value, int): + return value + s = str(value) + if s.isdigit(): + return int(s) + if len(s) == 16: + log.warning("Encoded ID where unencoded ID expected.") + return int(security.decode_id(s)) + raise ParameterValueError(f"invalid dataset id {value!r}", parameter_name) + + def src_id_to_item( sa_session: "Session", value: typing.MutableMapping[str, Any], security: "IdEncodingHelper" ) -> ItemFromSrcAny: @@ -2292,14 +2311,8 @@ class DataToolParameter(BaseDataToolParameter): ): rval.append(single_value) else: - if len(str(single_value)) == 16: - # Could never really have an ID this big anyway - postgres doesn't - # support that for integer column types. - log.warning("Encoded ID where unencoded ID expected.") - single_value = trans.security.decode_id(single_value) - if single_value is None: - raise ParameterValueError("unexpected None in data parameter value list", self.name) - rval.append(trans.sa_session.get(HistoryDatasetAssociation, single_value)) + pk = _decode_dataset_id(single_value, trans.security, self.name) + rval.append(trans.sa_session.get(HistoryDatasetAssociation, pk)) if len(found_srcs) > 1 and "hdca" in found_srcs: raise ParameterValueError( "if collections are supplied to multiple data input parameter, only collections may be used", @@ -2319,7 +2332,8 @@ class DataToolParameter(BaseDataToolParameter): elif isinstance(value, HistoryDatasetCollectionAssociation) or isinstance(value, DatasetCollectionElement): rval.append(value) else: - rval.append(session.get(HistoryDatasetAssociation, int(value))) + pk = _decode_dataset_id(value, trans.security, self.name) + rval.append(session.get(HistoryDatasetAssociation, pk)) dataset_matcher_factory = get_dataset_matcher_factory(trans) dataset_matcher = dataset_matcher_factory.dataset_matcher(self, other_values) for v in rval: diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index 2b5567af9d8..8752473d7b4 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -873,8 +873,6 @@ class DynamicOptions: by_dbkey.update(table_entries) for data_table_entry in by_dbkey.values(): field_entry = [] - if hda := data_table_entry.get("__hda__"): - field_entry.append(hda) missing_columns = False for column_key in self.tool_data_table.columns.keys(): if column_key not in data_table_entry: @@ -885,6 +883,13 @@ class DynamicOptions: break field_entry.append(data_table_entry[column_key]) if not missing_columns: + # The HDA must be appended after the columns: get_options() + # reads it from ``fields[-1]`` and indexes the other columns + # by their declared positions. Prepending shifts every + # column by one and leaks the HDA into the option ``value`` + # (#22674). + if hda := data_table_entry.get("__hda__"): + field_entry.append(hda) fields.append(field_entry) return fields diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 117fe123ac0..eacbba661b8 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -174,8 +174,9 @@ class Repeat(Group): rval = [] for i in range(self.default): rval_dict = {"__index__": i} + child_context = ExpressionContext(rval_dict, context) for input in self.inputs.values(): - rval_dict[input.name] = input.get_initial_value(trans, context) + rval_dict[input.name] = input.get_initial_value(trans, child_context) rval.append(rval_dict) return rval diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 07c89b89f67..ef9898661f6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -17,6 +17,7 @@ from galaxy.util import ( unicodify, ) from galaxy.util.hash_util import hmac_new +from galaxy.web import url_for from galaxy.webapps.base.controller import BaseUIController log = logging.getLogger(__name__) @@ -230,6 +231,6 @@ class ASync(BaseUIController): trans.sa_session.commit() - return trans.show_ok_message( - "A job has been successfully added to the queue. You can check the status of queued jobs in the History panel." - ) + # Return the user to the Galaxy SPA; the frontend surfaces a toast + # based on the `notification` query parameter. + return trans.response.send_redirect(url_for("/?notification=tool-submitted")) diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index 40b12cbd2ce..c5112e63390 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -122,16 +122,13 @@ class ToolRunner(BaseUIController): error(galaxy.util.unicodify(e)) if len(params) > 0: trans.log_event(f"Tool params: {str(params)}", tool_id=tool_id) - status_text = "You can check the status of queued jobs in the History panel." if job_errors := vars.get("job_errors"): errors = "\n".join(f"- {job_error}" for job_error in job_errors) message = f"There were errors setting up {len(job_errors)} submitted job(s):\n{errors}" return trans.show_error_message(message) - if (num_jobs := vars.get("num_jobs")) > 1: - message = f"{num_jobs} jobs have been successfully added to the queue. {status_text}" - else: - message = f"A job has been successfully added to the queue. {status_text}" - return trans.show_ok_message(message) + # Return the user to the Galaxy SPA; the frontend surfaces a toast + # based on the `notification` query parameter. + return trans.response.send_redirect(url_for("/?notification=tool-submitted")) @web.expose def rerun(self, trans, id=None, job_id=None, **kwd): diff --git a/lib/galaxy/webapps/galaxy/services/help.py b/lib/galaxy/webapps/galaxy/services/help.py index e75bf087e68..ffc317a5afa 100644 --- a/lib/galaxy/webapps/galaxy/services/help.py +++ b/lib/galaxy/webapps/galaxy/services/help.py @@ -1,7 +1,12 @@ import logging from galaxy.config import GalaxyAppConfiguration -from galaxy.exceptions import ServerNotConfiguredForRequest +from galaxy.exceptions import ( + GatewayTimeoutException, + InternalServerError, + ServerNotConfiguredForRequest, + UpstreamProxyError, +) from galaxy.schema.help import HelpForumSearchResponse from galaxy.security.idencoding import IdEncodingHelper from galaxy.util import requests @@ -34,10 +39,35 @@ class HelpService(ServiceBase): if not self.config.help_forum_api_url: raise ServerNotConfiguredForRequest("Help forum API URL is not configured.") forum_search_url = f"{self.config.help_forum_api_url}/search.json" - response = requests.get( - url=forum_search_url, - params={ - "q": query, - }, - ) - return HelpForumSearchResponse(**response.json()) + try: + response = requests.get( + url=forum_search_url, + params={ + "q": query, + }, + ) + except requests.exceptions.ConnectionError: + raise UpstreamProxyError( + "Could not connect to the Galaxy Help Forum. The service may be temporarily unavailable." + ) + except requests.exceptions.Timeout: + raise GatewayTimeoutException("The request to the Galaxy Help Forum timed out. Please try again later.") + except requests.exceptions.RequestException as e: + raise InternalServerError(f"An error occurred while requesting the Galaxy Help Forum: {e}") + + if not response.ok: + if 400 <= response.status_code < 500: + raise InternalServerError( + f"The Galaxy Help Forum returned an error (HTTP {response.status_code}). " + "This may indicate a misconfigured URL or API key that requires admin intervention." + ) + else: + raise UpstreamProxyError( + f"The Galaxy Help Forum returned an error (HTTP {response.status_code}). " + "The service may be temporarily unavailable. Please try again later." + ) + + try: + return HelpForumSearchResponse(**response.json()) + except ValueError as e: + raise InternalServerError(f"Received an unexpected response format from the Galaxy Help Forum: {e}") diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 03c53c1586f..118b6100c5d 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -131,6 +131,7 @@ from galaxy.util.json import safe_loads from galaxy.util.rules_dsl import RuleSet from galaxy.util.template import fill_template from galaxy.util.tool_shed.common_util import get_tool_shed_url_from_tool_shed_registry +from galaxy.util.tool_version import remove_version_from_guid from galaxy.workflow.workflow_parameter_input_definitions import ( get_default_parameter, INPUT_PARAMETER_TYPES, @@ -2326,6 +2327,13 @@ class ToolModule(WorkflowModule): if tool_version: tool_version = str(tool_version) tool_uuid = d.get("tool_uuid", None) + if tool_version == "latest": + # Resolve to the actual latest installed version via lineage rather than matching the exact GUID. + # This mirrors what &version=latest does in the tool form. + tool_version = None + if tool_id: + tool_id = remove_version_from_guid(tool_id) or tool_id + kwds = dict(kwds, exact_tools=False) if tool_id is None and tool_uuid is None: tool_representation = d.get("tool_representation") if tool_representation: diff --git a/lib/galaxy_test/api/test_authenticate.py b/lib/galaxy_test/api/test_authenticate.py index 68c5f25c855..dfce53de53f 100644 --- a/lib/galaxy_test/api/test_authenticate.py +++ b/lib/galaxy_test/api/test_authenticate.py @@ -42,13 +42,14 @@ class TestAuthenticateApi(ApiTestCase): tool_runner_response = get( urljoin(self.url, "tool_runner?tool_id=test_data_source"), cookies={"galaxytoolrunnersession": tool_runner_session_cookie}, + allow_redirects=False, ) - tool_runner_response.raise_for_status() + # On success, the controller redirects back to the SPA so the + # frontend can surface a "tool-submitted" toast. + assert tool_runner_response.status_code == 302 + assert "notification=tool-submitted" in tool_runner_response.headers["Location"] # Verify that we're not returning the sessioncookie assert "galaxysession" not in tool_runner_response.cookies - # Verify text message - text = tool_runner_response.text - assert "A job has been successfully added to the queue." in text # Make sure history for original session received job current_history_json_response = get( urljoin(self.url, "history/current_history_json"), cookies={"galaxysession": galaxy_session_cookie} diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 1558752dfd2..da9f7570238 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -388,6 +388,54 @@ steps: assert show_jobs_response.json()["external_id"] is not None assert show_jobs_response.json()["command_line"] is not None + @skip_without_tool("collection_creates_pair") + @requires_new_history + def test_show_collection_only_job_public(self, history_id): + # Regression test for https://github.com/galaxyproject/galaxy/issues/22602. + job_id, hdca_id = self._run_collection_only_job(history_id) + hdca = self.dataset_populator.get_history_collection_details(history_id, content_id=hdca_id) + for element in hdca["elements"]: + response = self.dataset_populator.make_dataset_public_raw(history_id, element["object"]["id"]) + assert_status_code_is_ok(response) + with self._different_user(anon=True): + show_jobs_response = self._get(f"jobs/{job_id}") + self._assert_status_code_is(show_jobs_response, 200) + assert show_jobs_response.json()["id"] == job_id + + @skip_without_tool("collection_creates_pair") + @requires_new_history + def test_show_collection_only_job_private_denied(self, history_id): + job_id, hdca_id = self._run_collection_only_job(history_id) + hdca = self.dataset_populator.get_history_collection_details(history_id, content_id=hdca_id) + for element in hdca["elements"]: + self.dataset_populator.make_private(history_id, element["object"]["id"]) + with self._different_user(): + show_jobs_response = self._get(f"jobs/{job_id}") + self._assert_status_code_is(show_jobs_response, 403) + + @requires_new_history + def test_show_job_accessible_via_public_history(self, history_id): + self.__history_with_new_dataset(history_id) + jobs_response = self._get("jobs", data={"history_id": history_id}) + job_id = jobs_response.json()[0]["id"] + self.dataset_populator.make_public(history_id) + with self._different_user(): + show_jobs_response = self._get(f"jobs/{job_id}") + self._assert_status_code_is(show_jobs_response, 200) + assert show_jobs_response.json()["id"] == job_id + + def _run_collection_only_job(self, history_id): + input_id = self.dataset_populator.new_dataset(history_id, content="a\nb\nc\nd\n", wait=True)["id"] + run_response = self.dataset_populator.run_tool( + tool_id="collection_creates_pair", + inputs={"input1": {"src": "hda", "id": input_id}}, + history_id=history_id, + ) + job_id = run_response["jobs"][0]["id"] + self.dataset_populator.wait_for_job(job_id, assert_ok=True) + hdca_id = run_response["output_collections"][0]["id"] + return job_id, hdca_id + def _run_detect_errors(self, history_id, inputs): payload = self.dataset_populator.run_tool_payload( tool_id="detect_errors_aggressive", diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py index c85f67aaa95..add8aa54d50 100644 --- a/lib/galaxy_test/api/test_tool_execute.py +++ b/lib/galaxy_test/api/test_tool_execute.py @@ -264,6 +264,27 @@ def test_map_over_empty_collection(target_history: TargetHistory, required_tool: assert "on collection 1" in name +@requires_tool_id("collection_paired_structured_like_with_data_input") +def test_map_over_empty_with_structured_like_non_mapped_collection_input( + target_history: TargetHistory, required_tool: RequiredTool +): + # Regression guard: an output declared ``structured_like=`` must precreate an implicit output even when the + # mapped-over input is empty (zero jobs). Before the fix, + # example_params fell back to param_template where the non-mapped + # collection's batch wrapper was never substituted, and precreate + # crashed with "Referenced input parameter is not a collection." + empty_hdca = target_history.with_list([]) + shape_hdca = target_history.with_pair(["a", "b"]) + inputs = { + "input1": {"batch": True, "values": [empty_hdca.src_dict]}, + "shape": shape_hdca.src_dict, + } + execute = required_tool.execute().with_inputs(inputs) + execute.assert_has_n_jobs(0) + execute.assert_creates_implicit_collection(0) + + @dataclass class MultiRunInRepeatFixtures: repeat_datasets: list[SrcDict] diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index a68bac54963..b247e732f74 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -4011,5 +4011,51 @@ class TestToolsApi(ApiTestCase, TestsTools): yield other_history_id +class TestDataManagerToolsApi(ApiTestCase, TestsTools): + """API tests that need the test case to act as an admin (e.g. data managers).""" + + require_admin_user = True + dataset_populator: DatasetPopulator + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) + + def test_build_does_not_leak_hda_from_user_bundle(self): + # Regression for https://github.com/galaxyproject/galaxy/issues/22674 + # When the user already has a ``data_manager_json`` bundle in their + # history for a tool data table, ``DynamicOptions.get_user_options`` + # builds a synthetic option row from it. A 2025-02-18 refactor + # (172ef05f269) prepended the bundle's HDA to that row, shifting every + # declared column index by one and leaking the raw HDA into the + # option's ``value`` field. ``/api/tools/{id}/build`` then 500s with + # TypeError: Object of type HistoryDatasetAssociation is not JSON serializable + # Reproduce by producing a real bundle (data_manager_mode=bundle), then + # loading the same tool's form. + history_id = self.dataset_populator.new_history() + payload = self.dataset_populator.run_tool_payload( + tool_id="data_manager_select", + inputs={"index": "hg19_value"}, + data_manager_mode="bundle", + history_id=history_id, + ) + create_response = self.dataset_populator._post("tools", data=payload) + create_response.raise_for_status() + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + dataset = self.dataset_populator.get_history_dataset_details(history_id) + assert dataset["extension"] == "data_manager_json" + + build = self.dataset_populator.build_tool_state("data_manager_select", history_id, inputs={}) + index_input = next(i for i in build["inputs"] if i["name"] == "index") + option_names = [option[0] for option in index_input["options"]] + # On-disk fasta_indexes.loc entries... + assert "hg19_name" in option_names + # ...plus the synthetic option contributed by the user's bundle. Before + # the fix the bundle's HDA was wired into ``value`` instead of + # ``dataset`` and JSON encoding crashed before this assertion ran. + assert "regression_name" in option_names + + def dataset_to_param(dataset): return dict(src="hda", id=dataset["id"]) diff --git a/lib/galaxy_test/selenium/test_data_source_tools.py b/lib/galaxy_test/selenium/test_data_source_tools.py index 8728a7e6485..47d81fef129 100644 --- a/lib/galaxy_test/selenium/test_data_source_tools.py +++ b/lib/galaxy_test/selenium/test_data_source_tools.py @@ -12,6 +12,7 @@ from .framework import ( class TestDataSource(SeleniumTestCase, UsesHistoryItemAssertions): ensure_registered = True + framework_tool_and_types = True @pytest.mark.skip("Skipping UCSC table direct1 data source test, chromedriver fails captcha") @selenium_test @@ -38,3 +39,20 @@ class TestDataSource(SeleniumTestCase, UsesHistoryItemAssertions): self.history_panel_wait_for_hid_ok(1, allowed_force_refreshes=2) # Make sure we're still logged in (xref https://github.com/galaxyproject/galaxy/issues/11374) self.components.masthead.logged_in_only.wait_for_visible() + + @selenium_test + @managed_history + def test_tool_runner_redirects_to_spa(self): + """Data source tools redirect back to the Galaxy SPA after handing off control to the controller. + + Regression test for https://github.com/galaxyproject/galaxy/issues/22671: the new tab opened + by an external data source app used to hit `/tool_runner?tool_id=...` and then JS-redirect + back to `/`. After the mako removal the new tab was stranded on a static "ok" page; the + controller now sends a 302 to `/?notification=tool-submitted` and the SPA surfaces a toast. + """ + self.get("tool_runner?tool_id=test_data_source") + # If the redirect is broken we land on a static page with no masthead. + self.wait_for_masthead() + # Confirm the SPA queued a toast from the notification query param. + self.wait_for_selector_visible(".b-toast") + self.screenshot("tool_runner_redirect_toast") diff --git a/lib/galaxy_test/selenium/test_workflow_rerun.py b/lib/galaxy_test/selenium/test_workflow_rerun.py index 8c13a10baa5..7b9936a594f 100644 --- a/lib/galaxy_test/selenium/test_workflow_rerun.py +++ b/lib/galaxy_test/selenium/test_workflow_rerun.py @@ -1,4 +1,7 @@ -from galaxy_test.base.workflow_fixtures import WORKFLOW_SIMPLE_CAT_TWICE +from galaxy_test.base.workflow_fixtures import ( + WORKFLOW_OPTIONAL_TRUE_INPUT_DATA, + WORKFLOW_SIMPLE_CAT_TWICE, +) from .framework import ( managed_history, retry_assertion_during_transitions, @@ -134,6 +137,46 @@ class TestWorkflowRun(SeleniumTestCase, UsesHistoryItemAssertions, RunsWorkflows self._assert_input_table_has_parameter("bool_param", "true") self.screenshot("workflow_rerun_boolean_inputs_tab") + @selenium_test + @managed_history + def test_workflow_rerun_with_unset_optional_data_input(self): + """Regression test for workflow rerun crashing when an optional data_input was left empty. + + When a workflow has an optional ``data_input`` step that the user did + not provide on the original run, ``WorkflowInvocationRequestModel.inputs`` + returns ``null`` for that step. WorkflowRunFormSimple.vue used to wrap + the value as ``{values: [null]}`` regardless, which crashed the + ``FormData`` mounted hook on ``"src" in null``. Because the mounted + hook never completed, the bad wrapper survived in formData and was + sent to the server, which rejected it with a ``DataOrCollectionRequest`` + union ValidationError ("Extra inputs are not permitted in 'values'"). + Reported on a "1 or 2 haplotypes" workflow where the second haplotype + was left empty. + """ + invocations = self.components.invocations + + # Upload one HDA so the history isn't empty (the workflow form needs a + # current history); the optional input is intentionally left unselected. + self.perform_upload(self.get_filename("1.fasta")) + self.wait_for_history() + self.workflow_run_open_workflow(WORKFLOW_OPTIONAL_TRUE_INPUT_DATA) + self.workflow_run_submit() + self.sleep_for(self.wait_types.UX_TRANSITION) + self.workflow_run_wait_for_ok(hid=2, expand=True) + + # Submitting lands us on the new invocation page; click rerun directly. + invocations.state_details.wait_for_visible() + invocations.workflow_rerun_button.wait_for_and_click() + self.sleep_for(self.wait_types.UX_TRANSITION) + + # Submit the rerun without touching anything. Pre-fix this raises a + # "Workflow submission failed: ... 'values' not permitted ..." toast + # because formData carries the poisoned `{values: [null]}` wrapper. + self.workflow_run_submit() + self.sleep_for(self.wait_types.UX_TRANSITION) + self.workflow_run_wait_for_ok(hid=3, expand=True) + self.screenshot("workflow_rerun_unset_optional_data_input_submitted") + @retry_assertion_during_transitions def _assert_input_table_has_parameter(self, label: str, value: str): table = self.wait_for_selector('[data-description="input table"]') diff --git a/test/functional/tools/collection_paired_structured_like_with_data_input.xml b/test/functional/tools/collection_paired_structured_like_with_data_input.xml new file mode 100644 index 00000000000..23a6647a439 --- /dev/null +++ b/test/functional/tools/collection_paired_structured_like_with_data_input.xml @@ -0,0 +1,13 @@ + + '${list_output.forward}'; + cat '$input1' '${shape.reverse}' '${shape.forward}' > '${list_output.reverse}' + ]]> + + + + + + + + diff --git a/test/functional/tools/data_manager_select.xml b/test/functional/tools/data_manager_select.xml new file mode 100644 index 00000000000..31652eca004 --- /dev/null +++ b/test/functional/tools/data_manager_select.xml @@ -0,0 +1,19 @@ + + Mirrors twobit_builder shape: select sourced from a tool data table. Regression for #22674. + + {"data_tables": {"test_fasta_indexes": [{"value": "regression_value", "dbkey": "regression_dbkey", "name": "regression_name", "path": "regression.fa"}]}} + + '$out_file.files_path/regression.fa'; + cp '$static_test_data' '$out_file' + ]]> + + + + + + + + + diff --git a/test/functional/tools/sample_data_manager_conf.xml b/test/functional/tools/sample_data_manager_conf.xml index e325f6b1381..b5ed89e74e1 100644 --- a/test/functional/tools/sample_data_manager_conf.xml +++ b/test/functional/tools/sample_data_manager_conf.xml @@ -13,4 +13,14 @@ + + + + + + + + + + diff --git a/test/functional/tools/sample_tool_conf.xml b/test/functional/tools/sample_tool_conf.xml index c3a9bf6f92d..da739aee215 100644 --- a/test/functional/tools/sample_tool_conf.xml +++ b/test/functional/tools/sample_tool_conf.xml @@ -190,6 +190,7 @@ + diff --git a/test/integration/test_structured_like_unpopulated.py b/test/integration/test_structured_like_unpopulated.py new file mode 100644 index 00000000000..e3e2a560df2 --- /dev/null +++ b/test/integration/test_structured_like_unpopulated.py @@ -0,0 +1,93 @@ +"""Integration test for ToolInputsNotReady on structured_like/unpopulated input. + +When a tool output is ``structured_like=""`` and +the user maps the tool over an empty collection, implicit output collection +precreation consults that input to determine output shape. If the referenced +collection is still populating, we should raise ``ToolInputsNotReadyException`` +(HTTP 400, ``TOOL_INPUTS_NOT_READY``) rather than surfacing the cryptic +"Referenced input parameter is not a collection." (Sentry issue GALAXY-MAIN-4KSCZZZ0015NC). + +This test deterministically produces an unpopulated DatasetCollection by +downgrading ``populated_state`` directly in the DB after the collection has +been created via the standard fetch path — the only reliable way to simulate +the race-window state from pure API tests. +""" + +from sqlalchemy import select + +from galaxy.model import ( + DatasetCollection, + HistoryDatasetCollectionAssociation, +) +from galaxy_test.base.populators import ( + DatasetCollectionPopulator, + DatasetPopulator, +) +from galaxy_test.driver import integration_util + + +class TestStructuredLikeUnpopulatedRaisesNotReady(integration_util.IntegrationTestCase): + framework_tool_and_types = True + + dataset_populator: DatasetPopulator + dataset_collection_populator: DatasetCollectionPopulator + require_admin_user = True + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) + + @property + def sa_session(self): + return self._app.model.session + + def _mark_collection_unpopulated(self, hdca_id: str) -> None: + hdca_db_id = self._get(f"configuration/decode/{hdca_id}").json()["decoded_id"] + # HDCA.collection_id points at the DatasetCollection row we need. + hdca_model = self.sa_session.scalar( + select(HistoryDatasetCollectionAssociation).where(HistoryDatasetCollectionAssociation.id == hdca_db_id) + ) + assert hdca_model is not None + dc_model = hdca_model.collection + dc_model.populated_state = DatasetCollection.populated_states.NEW + self.sa_session.add(dc_model) + self.sa_session.commit() + + def test_unpopulated_structured_like_target_raises_not_ready(self): + with self.dataset_populator.test_history() as history_id: + empty_hdca = self.dataset_collection_populator.create_list_in_history( + history_id, contents=[], direct_upload=True, wait=True + ).json()["output_collections"][0] + + shape_response = self.dataset_collection_populator.create_pair_in_history( + history_id, contents=["a", "b"], direct_upload=True, wait=True + ).json() + shape_hdca = shape_response["output_collections"][0] + + # Simulate upstream "still populating" — mirrors what + # happens when the referenced collection is an implicit + # collection whose producing jobs haven't finished yet. + self._mark_collection_unpopulated(shape_hdca["id"]) + + inputs = { + "input1": {"batch": True, "values": [{"src": "hdca", "id": empty_hdca["id"]}]}, + "shape": {"src": "hdca", "id": shape_hdca["id"]}, + } + response = self.dataset_populator.run_tool_raw( + tool_id="collection_paired_structured_like_with_data_input", + inputs=inputs, + history_id=history_id, + ) + + # Expect HTTP 400 (ToolInputsNotReadyException) with the + # same message meta.py raises for mapped-over unpopulated + # HDCAs, not the cryptic "Referenced input parameter..." + # that used to reach Sentry. + assert ( + response.status_code == 400 + ), f"Expected 400 for unpopulated input collection, got {response.status_code}: {response.text}" + assert "not populated" in response.text, f"Expected 'not populated' in error body, got: {response.text}" + assert ( + "Referenced input parameter is not a collection" not in response.text + ), "Regression: old cryptic error surfaced instead of ToolInputsNotReady" diff --git a/test/unit/app/tools/test_data_parameters.py b/test/unit/app/tools/test_data_parameters.py index fc139e1089c..fda59b1df5e 100644 --- a/test/unit/app/tools/test_data_parameters.py +++ b/test/unit/app/tools/test_data_parameters.py @@ -3,8 +3,11 @@ from typing import ( Optional, ) +import pytest + from galaxy import model from galaxy.app_unittest_utils import galaxy_mock +from galaxy.tools.parameters.basic import ParameterValueError from .util import BaseParameterTestCase @@ -33,6 +36,15 @@ class TestDataToolParameter(BaseParameterTestCase): # to just filter it out. assert [hda] == self.param.to_python(f"{hda.id},None", self.app) + def test_from_json_rejects_src_prefixed_string(self): + bogus = "hda:f9cad7b01a472135e2c8f5464c5c5ecb" + with pytest.raises(ParameterValueError, match="invalid dataset id"): + self.param.from_json([bogus], self.trans) + + def test_from_json_rejects_garbage_string(self): + with pytest.raises(ParameterValueError, match="invalid dataset id"): + self.param.from_json("not-an-id", self.trans) + def test_field_filter_on_types(self): hda1 = MockHistoryDatasetAssociation(name="hda1", id=1) hda2 = MockHistoryDatasetAssociation(name="hda2", id=2) diff --git a/test/unit/files/test_webdav.py b/test/unit/files/test_webdav.py index f50296b84bf..20a15993803 100644 --- a/test/unit/files/test_webdav.py +++ b/test/unit/files/test_webdav.py @@ -109,3 +109,20 @@ def test_serialization_user(): res = list_root(file_sources, "gxfiles://test1", recursive=True, user_context=None) assert find_file_a(res) + + +@skip_if_no_webdav +def test_url_preserved_in_serialization(): + # Regression test: 'url' is in COMMON_FILE_SOURCE_PROP_NAMES and was excluded from + # _serialize_config, causing WebDAVFS to be initialized with url=None, which led to + # AttributeError: 'NoneType' object has no attribute 'rstrip' when fetching files. + file_sources = configured_file_sources(FILE_SOURCES_CONF) + fs = file_source_as_webdav(file_sources._file_sources[0]) + + serialized = fs.to_dict(for_serialization=True) + assert "url" in serialized, "WebDAV url must be preserved in serialized form for job runner reconstruction" + assert serialized["url"] == "http://127.0.0.1:7083" + + recovered = serialize_and_recover(file_sources) + recovered_fs = file_source_as_webdav(recovered._file_sources[0]) + assert recovered_fs._get_runtime_context().config.url == "http://127.0.0.1:7083" diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py index f60e6b85ad2..6ef6f84911f 100644 --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -130,6 +130,33 @@ def test_cannot_create_tool_modules_for_missing_tools(): assert not module.tool +def test_tool_version_latest_resolves_toolshed_guid(): + # Toolshed GUIDs embed the version as the last segment. When tool_version="latest" + # is requested (as the WF editor does on insert), from_dict should strip the version + # from the GUID and resolve to the latest installed version via the versionless key. + trans = MockTrans() + old_guid = "toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.68+galaxy1" + versionless_guid = "toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc" + latest_tool = __mock_tool( + id="toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.74+galaxy1", version="0.74+galaxy1" + ) + trans.app.toolbox.tools[versionless_guid] = latest_tool + module = modules.module_factory.from_dict(trans, {"type": "tool", "content_id": old_guid, "tool_version": "latest"}) + assert module.tool is not None + assert module.tool.version == "0.74+galaxy1" + + +def test_tool_version_latest_resolves_builtin_tool(): + # Built-in tool IDs have no version segment; remove_version_from_guid returns None + # so the ID is unchanged. tool_version="latest" should still resolve correctly. + trans = MockTrans() + latest_tool = __mock_tool(id="cat1", version="2.0") + trans.app.toolbox.tools["cat1"] = latest_tool + module = modules.module_factory.from_dict(trans, {"type": "tool", "content_id": "cat1", "tool_version": "latest"}) + assert module.tool is not None + assert module.tool.version == "2.0" + + def test_updated_tool_version(): trans = MockTrans() mock_tool = __mock_tool(id="cat1", version="0.9")