diff --git a/client/package.json b/client/package.json index a23cf85bded..de6c7bdfc2b 100644 --- a/client/package.json +++ b/client/package.json @@ -199,6 +199,7 @@ "prettier": "^3.6.2", "sass": "^1.94.2", "store": "^2.0.12", + "timezone-mock": "^1.3.6", "typescript": "^5.7.3", "vite": "^7.2.4", "vite-tsconfig-paths": "^5.1.4", diff --git a/client/pnpm-lock.yaml b/client/pnpm-lock.yaml index ab6f6d9ed46..ff95cffa6fc 100644 --- a/client/pnpm-lock.yaml +++ b/client/pnpm-lock.yaml @@ -458,6 +458,9 @@ importers: store: specifier: ^2.0.12 version: 2.0.12 + timezone-mock: + specifier: ^1.3.6 + version: 1.4.2 typescript: specifier: ^5.7.3 version: 5.8.3 @@ -3965,6 +3968,9 @@ packages: resolution: {integrity: sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==} engines: {node: '>=0.6.0'} + timezone-mock@1.4.2: + resolution: {integrity: sha512-RgPQkSrARNHBnchuFpjrYf97uWgEaseCXK2asSeGcgUzRUO3cBR9Zu0uO1z2Eyi/TQsQtHIvQLi0br30COHWUA==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -8313,6 +8319,8 @@ snapshots: dependencies: setimmediate: 1.0.5 + timezone-mock@1.4.2: {} + tinybench@2.9.0: {} tinyexec@0.3.2: {} diff --git a/client/src/components/ActivityBar/ActivitySettings.vue b/client/src/components/ActivityBar/ActivitySettings.vue index 9113d5eb092..91033c22fce 100644 --- a/client/src/components/ActivityBar/ActivitySettings.vue +++ b/client/src/components/ActivityBar/ActivitySettings.vue @@ -110,28 +110,30 @@ function executeActivity(activity: Activity) { @click.stop="onRemove(activity)"> - - - - - - + diff --git a/client/src/components/UtcDate.vue b/client/src/components/UtcDate.vue index 09a51561601..8d1e1dd0d17 100644 --- a/client/src/components/UtcDate.vue +++ b/client/src/components/UtcDate.vue @@ -1,7 +1,9 @@ diff --git a/client/src/utils/dates.test.ts b/client/src/utils/dates.test.ts new file mode 100644 index 00000000000..e7b6f45318a --- /dev/null +++ b/client/src/utils/dates.test.ts @@ -0,0 +1,52 @@ +import MockDate from "timezone-mock"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { formatGalaxyPrettyDateString, galaxyTimeToDate, localizeUTCPretty } from "./dates"; + +describe("dates.ts", () => { + beforeEach(() => { + MockDate.register("Etc/GMT+4"); + }); + + afterEach(() => { + MockDate.unregister(); + }); + + describe("galaxyTimeToDate", () => { + it("should convert valid galaxyTime string to Date object", () => { + const galaxyTime = "2023-10-01T12:00:00"; + const date = galaxyTimeToDate(galaxyTime); + expect(date).toBeInstanceOf(Date); + expect(date.toISOString()).toBe("2023-10-01T12:00:00.000Z"); + }); + + it("should append Z if missing and parse correctly", () => { + const galaxyTime = "2023-10-01T12:00:00"; + const date = galaxyTimeToDate(galaxyTime); + expect(date.toISOString()).toBe("2023-10-01T12:00:00.000Z"); + }); + + it("should throw an error for invalid galaxyTime string", () => { + const invalidGalaxyTime = "invalid-date-string"; + expect(() => galaxyTimeToDate(invalidGalaxyTime)).toThrow( + `Invalid galaxyTime string: ${invalidGalaxyTime}`, + ); + }); + }); + + describe("localizeUTCPretty", () => { + it("should format Date object into human-readable string", () => { + const date = new Date("2023-10-01T12:00:00Z"); + const formatted = localizeUTCPretty(date); + expect(formatted).toBe("Sunday Oct 1st 8:00:00 2023 GMT-4"); + }); + }); + + describe("formatGalaxyPrettyDateString", () => { + it("should convert galaxyTime string to formatted date string", () => { + const galaxyTime = "2023-10-01T12:00:00"; + const formatted = formatGalaxyPrettyDateString(galaxyTime); + expect(formatted).toBe("Sunday Oct 1st 8:00:00 2023 GMT-4"); + }); + }); +}); diff --git a/client/src/utils/dates.ts b/client/src/utils/dates.ts new file mode 100644 index 00000000000..7903a56561b --- /dev/null +++ b/client/src/utils/dates.ts @@ -0,0 +1,39 @@ +import { format, parseISO } from "date-fns"; + +/** + * Converts a Galaxy time string to a Date object. + * @param {string} galaxyTime - The Galaxy time string in ISO format. + * @returns {Date} The parsed Date object. + */ +export function galaxyTimeToDate(galaxyTime: string): Date { + // Galaxy doesn't include Zulu time zone designator, but it's always UTC + // so we need to add it to parse the string correctly in JavaScript. + let time = galaxyTime; + if (!time.endsWith("Z")) { + time += "Z"; + } + const date = parseISO(time); + if (isNaN(date.getTime())) { + throw new Error(`Invalid galaxyTime string: ${galaxyTime}`); + } + return date; +} + +/** + * Formats a UTC Date object into a human-readable string, localized to the user's time zone. + * @param {Date} utcDate - The UTC Date object. + * @returns {string} The formatted date string. + */ +export function localizeUTCPretty(utcDate: Date): string { + return format(utcDate, "eeee MMM do H:mm:ss yyyy zz"); +} + +/** + * Converts a Galaxy time string to a human-readable formatted date string, localized to the user's time zone. + * @param {string} galaxyTime - The Galaxy time string in ISO format. + * @returns {string} The formatted date string. + */ +export function formatGalaxyPrettyDateString(galaxyTime: string): string { + const date = galaxyTimeToDate(galaxyTime); + return localizeUTCPretty(date); +} diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 44e42f5cb68..cf5d2763861 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -620,9 +620,7 @@ class ReadyForExportMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHand pass def handle_invocation_time(self, line, invocation): - self.ensure_rendering_data_for("invocations", invocation)["create_time"] = _database_time_to_str( - invocation.create_time - ) + self.ensure_rendering_data_for("invocations", invocation)["create_time"] = invocation.create_time.isoformat() def handle_invocation_inputs(self, line, invocation): pass @@ -677,6 +675,9 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): def __init__(self, trans): self.trans = trans + def _format_printable_time(self, time): + return time.strftime("%Y-%m-%d, %H:%M:%S UTC") + def handle_dataset_display(self, line, hda): name = hda.name or "" markdown = "---\n" @@ -852,7 +853,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): return (content, True) def handle_generate_time(self, line, generate_time): - content = literal_via_fence(generate_time.isoformat()) + content = literal_via_fence(self._format_printable_time(generate_time)) return (content, True) def handle_instance_access_link(self, line, url): @@ -884,7 +885,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): return (content, True) def handle_invocation_time(self, line, invocation): - content = literal_via_fence(_database_time_to_str(invocation.create_time)) + content = literal_via_fence(self._format_printable_time(invocation.create_time)) return (content, True) def handle_invocation_inputs(self, line, invocation): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7b3cb75ea03..3554d14e665 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -59,6 +59,7 @@ from social_core.storage import ( from sqlalchemy import ( alias, and_, + any_, asc, BigInteger, bindparam, @@ -158,6 +159,7 @@ from galaxy.model.custom_types import ( UUIDType, ) from galaxy.model.database_object_names import NAMING_CONVENTION +from galaxy.model.database_utils import supports_skip_locked as _check_supports_skip_locked from galaxy.model.item_attrs import ( get_item_annotation_str, UsesAnnotations, @@ -4801,6 +4803,8 @@ class Dataset(Base, StorableObject, Serializable): dialect_name = session.bind.dialect.name if dialect_name in ("postgresql", "sqlite"): + if dialect_name == "postgresql" and supports_skip_locked is None: + supports_skip_locked = _check_supports_skip_locked(session.get_bind()) self._touch_collection_update_time_cte(session, supports_skip_locked) else: # Fallback for other databases @@ -7066,49 +7070,169 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): dc = alias(DatasetCollection.__table__) dce = alias(DatasetCollectionElement.__table__) depth_collection_type = dataset_collection.collection_type - order_by_columns = [dce.c.element_index] nesting_level = 0 def attribute_columns(column_collection, attributes, nesting_level=None): label_fragment = f"_{nesting_level}" if nesting_level is not None else "" return [getattr(column_collection, a).label(f"{a}{label_fragment}") for a in attributes] - q = ( - select( - *attribute_columns(dce.c, element_attributes, nesting_level), - *attribute_columns(dc.c, collection_attributes, nesting_level), + # On PostgreSQL, joining dataset_collection_element to itself causes + # the planner to choose catastrophically slow hash/merge joins: it + # overestimates cardinality and scans the entire table at each level. + # The ARRAY(subquery) pattern avoids all DCE-to-DCE joins by + # evaluating each nesting level into a concrete array before the next + # lookup, forcing index scans throughout. + # + # Per-level element attributes and hierarchical ordering are + # provided via correlated scalar subqueries that walk UP from each + # leaf DCE to its ancestors via the indexed child_collection_id + # column. Each subquery returns exactly one row via an index lookup. + # + # collection_attributes (e.g. populated_state) must stay on the + # outerjoin path: the ARRAY walk only visits branches that contain + # leaf DCEs, so empty sub-collections — common when conditional + # workflow steps are skipped — would be invisible and their + # populated_state never checked. + try: + session = object_session(self) + except Exception: + session = None + is_postgres = session is not None and session.bind and session.bind.dialect.name == "postgresql" + use_array_walk = is_postgres and ":" in depth_collection_type and not collection_attributes + + if use_array_walk: + # Build nested ARRAY(subquery) expressions to walk the tree + # without any DCE-to-DCE joins. + dce_table = DatasetCollectionElement.__table__ + inner_dce = alias(dce_table) + child_ids_subq = select(inner_dce.c.child_collection_id).where( + inner_dce.c.dataset_collection_id == dataset_collection.id ) - .select_from(dce, dc) - .join(dce, dce.c.dataset_collection_id == dc.c.id) - .filter(dc.c.id == dataset_collection.id) - ) - - while ":" in depth_collection_type: - nesting_level += 1 - inner_dce = alias(DatasetCollectionElement.__table__) - inner_dc = alias(DatasetCollection.__table__) - order_by_columns.append(inner_dce.c.element_index) - q = q.outerjoin(inner_dce, inner_dce.c.dataset_collection_id == dce.c.child_collection_id) - if collection_attributes: - q = q.outerjoin(inner_dc, inner_dc.c.id == dce.c.child_collection_id) - q = q.add_columns( - *attribute_columns(inner_dc.c, collection_attributes, nesting_level), + n_intermediates = depth_collection_type.count(":") + for _ in range(n_intermediates - 1): + next_dce = alias(dce_table) + child_ids_subq = select(next_dce.c.child_collection_id).where( + next_dce.c.dataset_collection_id == any_(func.array(child_ids_subq.scalar_subquery())) ) - q = q.add_columns(*attribute_columns(inner_dce.c, element_attributes, nesting_level)) - dce = inner_dce - dc = inner_dc - depth_collection_type = depth_collection_type.split(":", 1)[1] + leaf_dce = alias(dce_table) + q = ( + select() + .select_from(leaf_dce) + .where(leaf_dce.c.dataset_collection_id == any_(func.array(child_ids_subq.scalar_subquery()))) + ) + n_levels = n_intermediates + 1 - if ( + # Build navigation chain of collection ID expressions from leaf + # to root. coll_ids[0] = leaf's collection, coll_ids[k] = k + # steps up from the leaf. Each entry after the first is a + # correlated scalar subquery that walks one level up via the + # indexed child_collection_id column. + coll_ids: list = [leaf_dce.c.dataset_collection_id] + for _i in range(n_intermediates): + nav = alias(dce_table) + coll_ids.append( + select(nav.c.dataset_collection_id) + .where(nav.c.child_collection_id == coll_ids[-1]) + .correlate(leaf_dce) + .limit(1) + .scalar_subquery() + ) + + # Add element attribute columns for each level (root first, + # leaf last) to match the column ordering of the outerjoin path. + for level in range(n_levels): + steps_up = n_levels - 1 - level + if steps_up == 0: + # Leaf level: direct column access + q = q.add_columns(*[getattr(leaf_dce.c, a).label(f"{a}_{level}") for a in element_attributes]) + else: + # Ancestor level: correlated subquery + for attr in element_attributes: + nav = alias(dce_table) + q = q.add_columns( + select(getattr(nav.c, attr)) + .where(nav.c.child_collection_id == coll_ids[steps_up - 1]) + .correlate(leaf_dce) + .limit(1) + .scalar_subquery() + .label(f"{attr}_{level}") + ) + + hda_join_col = leaf_dce.c.hda_id + dce_id_col = leaf_dce.c.id + + # Build hierarchical ORDER BY from root level to leaf level. + order_by_columns: list[ColumnElement] = [] + for level in range(n_levels): + steps_up = n_levels - 1 - level + if steps_up == 0: + order_by_columns.append(leaf_dce.c.element_index) + else: + nav = alias(dce_table) + order_by_columns.append( + select(nav.c.element_index) + .where(nav.c.child_collection_id == coll_ids[steps_up - 1]) + .correlate(leaf_dce) + .limit(1) + .scalar_subquery() + ) + elif ":" not in depth_collection_type: + # Single-level collection: simple join, no nesting issues. + q = ( + select( + *attribute_columns(dce.c, element_attributes, nesting_level), + *attribute_columns(dc.c, collection_attributes, nesting_level), + ) + .select_from(dce, dc) + .join(dce, dce.c.dataset_collection_id == dc.c.id) + .filter(dc.c.id == dataset_collection.id) + ) + hda_join_col = dce.c.hda_id + dce_id_col = dce.c.id + order_by_columns = [dce.c.element_index] + else: + # Nested collection on SQLite, or collection_attributes on + # PostgreSQL: use the original outerjoin approach. + order_by_columns = [dce.c.element_index] + q = ( + select( + *attribute_columns(dce.c, element_attributes, nesting_level), + *attribute_columns(dc.c, collection_attributes, nesting_level), + ) + .select_from(dce, dc) + .join(dce, dce.c.dataset_collection_id == dc.c.id) + .filter(dc.c.id == dataset_collection.id) + ) + while ":" in depth_collection_type: + nesting_level += 1 + inner_dce = alias(DatasetCollectionElement.__table__) + inner_dc = alias(DatasetCollection.__table__) + order_by_columns.append(inner_dce.c.element_index) + q = q.outerjoin(inner_dce, inner_dce.c.dataset_collection_id == dce.c.child_collection_id) + if collection_attributes: + q = q.outerjoin(inner_dc, inner_dc.c.id == dce.c.child_collection_id) + q = q.add_columns( + *attribute_columns(inner_dc.c, collection_attributes, nesting_level), + ) + q = q.add_columns(*attribute_columns(inner_dce.c, element_attributes, nesting_level)) + dce = inner_dce + dc = inner_dc + depth_collection_type = depth_collection_type.split(":", 1)[1] + hda_join_col = dce.c.hda_id + dce_id_col = dce.c.id + + needs_hda_join = bool( hda_attributes or dataset_attributes or dataset_permission_attributes - or return_entities - and not return_entities == (DatasetCollectionElement,) - ): - q = q.join(HistoryDatasetAssociation).join(Dataset) + or (return_entities and return_entities != (DatasetCollectionElement,)) + ) + + if needs_hda_join: + q = q.join(HistoryDatasetAssociation, HistoryDatasetAssociation.id == hda_join_col) + q = q.join(Dataset, Dataset.id == HistoryDatasetAssociation.dataset_id) if dataset_permission_attributes: - q = q.join(DatasetPermissions) + q = q.join(DatasetPermissions, DatasetPermissions.dataset_id == Dataset.id) q = ( q.add_columns(*attribute_columns(HistoryDatasetAssociation, hda_attributes)) .add_columns(*attribute_columns(Dataset, dataset_attributes)) @@ -7117,7 +7241,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): for entity in return_entities: q = q.add_columns(entity) if entity == DatasetCollectionElement: - q = q.filter(entity.id == dce.c.id) + q = q.filter(entity.id == dce_id_col) q = q.order_by(*order_by_columns) return q @@ -7224,14 +7348,55 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): if ":" not in self.collection_type: _populated_optimized = self.populated_state == DatasetCollection.populated_states.OK else: - stmt = self._build_nested_collection_attributes_stmt( - collection_attributes=("populated_state",), - ) session = required_object_session(self) - for row in session.execute(stmt): - if any(state not in (DatasetCollection.populated_states.OK, None) for state in row): - _populated_optimized = False - break + is_postgres = session.bind and session.bind.dialect.name == "postgresql" + if is_postgres: + # Query intermediate collection IDs directly using the + # ARRAY walk pattern, then check their populated_state. + # Unlike the leaf-DCE ARRAY walk in + # _build_nested_collection_attributes_stmt, this + # correctly handles empty sub-collections (which have + # no leaf DCEs but may still have non-OK state, e.g. + # from skipped conditional workflow steps). + dce_table = DatasetCollectionElement.__table__ + dc_table = DatasetCollection.__table__ + n_intermediates = self.collection_type.count(":") + + inner_dce = alias(dce_table) + child_ids_array = func.array( + select(inner_dce.c.child_collection_id) + .where(inner_dce.c.dataset_collection_id == self.id) + .scalar_subquery() + ) + level_conditions = [dc_table.c.id == any_(child_ids_array)] + + for _ in range(n_intermediates - 1): + next_dce = alias(dce_table) + child_ids_array = func.array( + select(next_dce.c.child_collection_id) + .where(next_dce.c.dataset_collection_id == any_(child_ids_array)) + .scalar_subquery() + ) + level_conditions.append(dc_table.c.id == any_(child_ids_array)) + + stmt = ( + select(literal(1)) + .select_from(dc_table) + .where( + or_(*level_conditions), + dc_table.c.populated_state != DatasetCollection.populated_states.OK, + ) + .limit(1) + ) + _populated_optimized = session.execute(stmt).first() is None + else: + stmt = self._build_nested_collection_attributes_stmt( + collection_attributes=("populated_state",), + ) + for row in session.execute(stmt): + if any(state not in (DatasetCollection.populated_states.OK, None) for state in row): + _populated_optimized = False + break self._populated_optimized = _populated_optimized return self._populated_optimized diff --git a/lib/galaxy/model/database_utils.py b/lib/galaxy/model/database_utils.py index c4615f24ca9..fa1cdea0f54 100644 --- a/lib/galaxy/model/database_utils.py +++ b/lib/galaxy/model/database_utils.py @@ -1,15 +1,12 @@ import sqlite3 from contextlib import contextmanager +from functools import lru_cache from typing import ( NewType, Optional, ) -from sqlalchemy import ( - create_engine, - select, - update, -) +from sqlalchemy import create_engine from sqlalchemy.engine import Engine from sqlalchemy.engine.url import make_url from sqlalchemy.orm import object_session @@ -20,7 +17,6 @@ from sqlalchemy.sql.expression import ( ) from galaxy.exceptions import ConfigurationError -from galaxy.model import Job DbUrl = NewType("DbUrl", str) @@ -147,19 +143,21 @@ def is_one_database(db1_url: str, db2_url: Optional[str]): return not (db1_url and db2_url and db1_url != db2_url) +@lru_cache(maxsize=1) def supports_returning(engine: Engine) -> bool: """ Return True if the database bound to `engine` supports the `RETURNING` SQL clause. """ - stmt = update(Job).where(Job.id == -1).values(create_time=None).returning(Job.id) + stmt = text("UPDATE job SET tool_id=abc WHERE false RETURNING id") return _statement_executed_without_error(stmt, engine) +@lru_cache(maxsize=1) def supports_skip_locked(engine: Engine) -> bool: """ Return True if the database bound to `engine` supports the `SKIP_LOCKED` parameter. """ - stmt = select(Job).where(Job.id == -1).with_for_update(skip_locked=True) + stmt = text("SELECT 1 FROM job WHERE false FOR UPDATE SKIP LOCKED") return _statement_executed_without_error(stmt, engine) diff --git a/lib/galaxy/tool_util/parameters/factory.py b/lib/galaxy/tool_util/parameters/factory.py index b183589dc4e..149d1ff1d80 100644 --- a/lib/galaxy/tool_util/parameters/factory.py +++ b/lib/galaxy/tool_util/parameters/factory.py @@ -213,8 +213,10 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool name=input_source.parse_name(), **_common_param_kwargs(input_source), ) - elif param_type == "data": - optional = input_source.parse_optional() + elif param_type in ("data", "hidden_data"): + # hidden_data is broken without optional="true" (job runner rejects + # missing datasets); only known user is cufflinks which sets it. + optional = input_source.parse_optional() if param_type == "data" else True multiple = input_source.get_bool("multiple", False) return DataParameterModel( type="data", diff --git a/lib/galaxy/tool_util/verify/parse.py b/lib/galaxy/tool_util/verify/parse.py index 3938ed3770a..264095e6bc9 100644 --- a/lib/galaxy/tool_util/verify/parse.py +++ b/lib/galaxy/tool_util/verify/parse.py @@ -324,7 +324,7 @@ def _process_raw_inputs( location = param_extra.get("location") if param_type != "text": param_value = split_if_str(param_value) - if param_type == "data": + if param_type in ("data", "hidden_data"): if location and input_source.get_bool("multiple", False): # We get the input/s from the location which can be a list of urls separated by commas locations = split_if_str(location) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index bdf23eb8c05..91217e072d7 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1533,6 +1533,10 @@ class ColumnListParameter(SelectToolParameter): # otherwise read first row - assume is a header with tab separated names if self.usecolnames: dataset = other_values.get(self.data_ref, None) + if isinstance(dataset, HistoryDatasetCollectionAssociation): + dataset = dataset.to_hda_representative() + if isinstance(dataset, DatasetCollectionElement): + dataset = dataset.first_dataset_instance() column_names = getattr(dataset, "metadata", None) and dataset.metadata.get_if_set("column_names") if column_names: try: @@ -2710,6 +2714,14 @@ class HiddenDataToolParameter(HiddenToolParameter, DataToolParameter): self.value = "None" self.type = "hidden_data" self.hidden = True + # hidden_data params are broken without optional="true" - the job runner's + # parameter validation rejects them when no dataset is provided. The only + # known tool using hidden_data (cufflinks) sets optional="true". + if not self.optional: + raise ParameterValueError( + 'hidden_data parameters must declare optional="true" to function correctly', + self.name, + ) class BaseJsonToolParameter(ToolParameter): diff --git a/lib/galaxy/tools/parameters/populate_model.py b/lib/galaxy/tools/parameters/populate_model.py index 2a6a676a0a2..90ffb5326e4 100644 --- a/lib/galaxy/tools/parameters/populate_model.py +++ b/lib/galaxy/tools/parameters/populate_model.py @@ -1,4 +1,3 @@ -import logging from typing import ( Any, ) @@ -6,8 +5,6 @@ from typing import ( from galaxy.util.expressions import ExpressionContext from .basic import ImplicitConversionRequired -log = logging.getLogger(__name__) - def populate_model(request_context, inputs, state_inputs, group_inputs: list[dict[str, Any]], other_values=None): """ @@ -50,6 +47,8 @@ def populate_model(request_context, inputs, state_inputs, group_inputs: list[dic elif input.type == "section": tool_dict = input.to_dict(request_context) populate_model(request_context, input.inputs, group_state, tool_dict["inputs"], other_values) + elif input.type == "upload_dataset": + tool_dict = input.to_dict(request_context) else: try: initial_value = input.get_initial_value(request_context, other_values) @@ -63,9 +62,6 @@ def populate_model(request_context, inputs, state_inputs, group_inputs: list[dic tool_dict = input.to_dict(request_context, other_values=other_values) # This hack leads client to display a text field tool_dict["textable"] = True - except Exception: - tool_dict = input.to_dict(request_context, other_values=other_values) - log.exception("tools::to_json() - Skipping parameter expansion '%s'", input.name) if input_index >= len(group_inputs): group_inputs.append(tool_dict) else: diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 86e1738efe7..93daa1c1c36 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -267,7 +267,7 @@ class CatchingIter: if not self.start_checker.response_started: self.start_checker("500 Internal Server Error", [("content-type", "text/html")], exc_info) - return response + return response.encode("utf-8", errors="ignore") def close(self): # This should at least print something to stderr if the diff --git a/lib/galaxy/web/framework/middleware/profile.py b/lib/galaxy/web/framework/middleware/profile.py index 672c90d4154..7bfeed86907 100644 --- a/lib/galaxy/web/framework/middleware/profile.py +++ b/lib/galaxy/web/framework/middleware/profile.py @@ -67,7 +67,7 @@ class ProfileMiddleware: prof = cProfile.Profile() prof.runctx("run_app()", globals(), locals()) # Build up body with stats - body = "".join(body) + body = b"".join(body) headers = catch_response[1] content_type = response.header_value(headers, "content-type") if not content_type.startswith("text/html"): @@ -77,7 +77,7 @@ class ProfileMiddleware: stats.strip_dirs() stats.sort_stats("time", "calls") output = pstats_as_html(stats, self.limit) - body += template % output + body += (template % output).encode("utf-8") return [body] diff --git a/lib/galaxy/web/framework/middleware/static.py b/lib/galaxy/web/framework/middleware/static.py index 432c6084189..b7f9da7b85f 100644 --- a/lib/galaxy/web/framework/middleware/static.py +++ b/lib/galaxy/web/framework/middleware/static.py @@ -54,7 +54,7 @@ class CacheableStaticURLParser(StaticURLParser): headers: list[tuple[str, str]] = [] ETAG.update(headers, mytime) start_response("304 Not Modified", headers) - return [""] # empty body + return [b""] # empty body app = FileApp(full) if self.cache_seconds: app.cache_control(max_age=int(self.cache_seconds)) diff --git a/lib/galaxy_test/api/conftest.py b/lib/galaxy_test/api/conftest.py index 0cae9e6426a..17c8f19026b 100644 --- a/lib/galaxy_test/api/conftest.py +++ b/lib/galaxy_test/api/conftest.py @@ -20,6 +20,11 @@ from galaxy_test.base.api_util import ( get_user_api_key, ) from galaxy_test.base.env import setup_keep_outdir +from galaxy_test.base.mock_http_server import ( + MockHTTPRequestHandler, + MockHttpServer, + start_mock_http_server, +) from galaxy_test.base.populators import ( check_missing_tool, DatasetCollectionPopulator, @@ -177,3 +182,15 @@ def _requires_marker_to_effective_tool_id(anonymous_galaxy_interactor, marker): all_tool_ids = get_tool_ids(anonymous_galaxy_interactor) tool_id = [t for t in any_of_tool_ids if t in all_tool_ids][0] return tool_id + + +@pytest.fixture(scope="session") +def mock_http_server(): + if os.environ.get("GALAXY_TEST_EXTERNAL"): + yield MockHttpServer(base_url=None, handler_class=None, is_remote=True) + else: + server, base_url = start_mock_http_server() + try: + yield MockHttpServer(base_url=base_url, handler_class=MockHTTPRequestHandler, is_remote=False) + finally: + server.shutdown() diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py index 17f74ab7d18..7a505e7346f 100644 --- a/lib/galaxy_test/api/test_dataset_collections.py +++ b/lib/galaxy_test/api/test_dataset_collections.py @@ -6,7 +6,6 @@ from urllib.parse import quote from galaxy.tool_util_models.sample_sheet import SampleSheetColumnDefinitions from galaxy.util import galaxy_root_path -from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base.api_asserts import ( assert_has_key, assert_object_id_error, @@ -917,16 +916,19 @@ class TestDatasetCollectionsApi(ApiTestCase): object0 = element0["object"] assert object0["state"] == "deferred" - @skip_if_github_down - def test_upload_collection_failed_expansion_url(self): + def test_upload_collection_failed_expansion_url(self, mock_http_server): with self.dataset_populator.test_history(require_new=False) as history_id: + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", + file_path="test-data/4.bed", + ) targets = [ { "destination": {"type": "hdca"}, "elements_from": "bagit", "collection_type": "list", "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", + "url": url, } ] payload = { diff --git a/lib/galaxy_test/api/test_datasets.py b/lib/galaxy_test/api/test_datasets.py index a97cfe5885b..2c4704245d9 100644 --- a/lib/galaxy_test/api/test_datasets.py +++ b/lib/galaxy_test/api/test_datasets.py @@ -10,7 +10,6 @@ from galaxy.model.unittest_utils.store_fixtures import ( TEST_SOURCE_URI, ) from galaxy.tool_util.verify.test_data import TestDataResolver -from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base.api_asserts import assert_has_keys from galaxy_test.base.decorators import ( requires_admin, @@ -879,11 +878,14 @@ class TestDatasetsApi(ApiTestCase): assert len(sources) == 1 assert sources[0]["source_uri"] == TEST_SOURCE_URI - @skip_if_github_down - def test_display_application_link(self, history_id): + def test_display_application_link(self, history_id, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + file_path="test-data/1.bam", + ) item = { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + "url": url, "ext": "bam", } output = self.dataset_populator.fetch_hda(history_id, item) diff --git a/lib/galaxy_test/api/test_libraries.py b/lib/galaxy_test/api/test_libraries.py index d328e353131..264530311fb 100644 --- a/lib/galaxy_test/api/test_libraries.py +++ b/lib/galaxy_test/api/test_libraries.py @@ -4,7 +4,6 @@ from galaxy.model.unittest_utils.store_fixtures import ( one_ld_library_model_store_dict, TEST_LIBRARY_NAME, ) -from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base import api_asserts from galaxy_test.base.decorators import requires_new_library from galaxy_test.base.populators import ( @@ -317,16 +316,19 @@ class TestLibrariesApi(ApiTestCase): self._assert_status_code_is(create_response, 400) assert create_response.json()["err_msg"] == "Requested extension 'xxx' unknown, cannot upload dataset." - @skip_if_github_down @requires_new_library - def test_fetch_failed_validation(self): + def test_fetch_failed_validation(self, mock_http_server): # Exception handling is really rough here - we should be creating a dataset in error instead # of just failing the job like this. history_id, library, destination = self._setup_fetch_to_folder("single_url") + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", + file_path="test-data/4.bed", + ) items = [ { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", + "url": url, "MD5": "37b59762b59fff860460522d271bc112", "name": "4.bed", } @@ -352,16 +354,19 @@ class TestLibrariesApi(ApiTestCase): dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed") assert dataset["state"] == "error", dataset - @skip_if_github_down @requires_new_library - def test_fetch_url_archive_to_folder(self): + def test_fetch_url_archive_to_folder(self, mock_http_server): history_id, library, destination = self._setup_fetch_to_folder("single_url") + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip", + file_path="test-data/4.bed.zip", + ) targets = [ { "destination": destination, "items_from": "archive", "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip", + "url": url, } ] payload = { diff --git a/lib/galaxy_test/api/test_proxy.py b/lib/galaxy_test/api/test_proxy.py index 815284a22a3..a9516cb09a6 100644 --- a/lib/galaxy_test/api/test_proxy.py +++ b/lib/galaxy_test/api/test_proxy.py @@ -7,11 +7,12 @@ from unittest.mock import ( import pytest import requests -from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base.populators import DatasetPopulator from ._framework import ApiTestCase -URL_TO_TEST = "https://github.com/galaxyproject/galaxy/raw/6e7427e8e2e070d4ab491a6cce5a59b6b6a62a08/test-data/4.bed.zip" +REMOTE_URL_TO_TEST = ( + "https://github.com/galaxyproject/galaxy/raw/6e7427e8e2e070d4ab491a6cce5a59b6b6a62a08/test-data/4.bed.zip" +) EXPECTED_HEADERS = { "content-length": "198", "content-type": "application/zip", @@ -28,27 +29,36 @@ class TestProxyApi(ApiTestCase): super().setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) - @skip_if_github_down - def test_proxy_head_request(self): - response = self._head(f"proxy?url={URL_TO_TEST}") + def _get_zip_url(self, mock_http_server): + return mock_http_server.get_url( + remote_url=REMOTE_URL_TO_TEST, + file_path="test-data/4.bed.zip", + content_type="application/zip", + support_head=True, + support_ranges=True, + ) + + def test_proxy_head_request(self, mock_http_server): + url = self._get_zip_url(mock_http_server) + response = self._head(f"proxy?url={url}") self._assert_status_code_is_ok(response) assert response.headers["content-length"] == EXPECTED_HEADERS["content-length"] assert response.headers["content-type"] == EXPECTED_HEADERS["content-type"] assert response.headers["accept-ranges"] == EXPECTED_HEADERS["accept-ranges"] - @skip_if_github_down - def test_proxy_get_request(self): - response = self._get(f"proxy?url={URL_TO_TEST}") + def test_proxy_get_request(self, mock_http_server): + url = self._get_zip_url(mock_http_server) + response = self._get(f"proxy?url={url}") self._assert_status_code_is_ok(response) assert response.headers["content-length"] == EXPECTED_HEADERS["content-length"] assert response.headers["content-type"] == EXPECTED_HEADERS["content-type"] assert response.headers["accept-ranges"] == EXPECTED_HEADERS["accept-ranges"] assert len(response.content) == int(EXPECTED_HEADERS["content-length"]) - @skip_if_github_down - def test_proxy_get_request_with_range(self): + def test_proxy_get_request_with_range(self, mock_http_server): + url = self._get_zip_url(mock_http_server) request_range = "bytes=0-3" - response = self._get(f"proxy?url={URL_TO_TEST}", headers={"range": request_range}) + response = self._get(f"proxy?url={url}", headers={"range": request_range}) self._assert_status_code_is(response, 206) assert response.headers["accept-ranges"] == "bytes" assert response.headers["content-length"] == "4" @@ -56,7 +66,7 @@ class TestProxyApi(ApiTestCase): assert response.content == ZIP_MAGIC_NUMBER def test_anonymous_user_cannot_access_proxy(self): - response = self._get(f"proxy?url={URL_TO_TEST}", anon=True) + response = self._get(f"proxy?url={REMOTE_URL_TO_TEST}", anon=True) self._assert_status_code_is(response, 403) assert response.json()["err_msg"] == "Anonymous users are not allowed to access this endpoint" @@ -83,12 +93,12 @@ class TestProxyApi(ApiTestCase): @pytest.mark.parametrize( "url_with_nonprintable", [ - f"\t{URL_TO_TEST}", # Tab at beginning - f"{URL_TO_TEST}\t", # Tab at end + f"\t{REMOTE_URL_TO_TEST}", # Tab at beginning + f"{REMOTE_URL_TO_TEST}\t", # Tab at end "https://example.com/path\twith\ttab", # Tabs in path "https://exam\tple.com/path", # Tab in domain - f"\r\n{URL_TO_TEST}", # CRLF at beginning - f"{URL_TO_TEST}\r\n", # CRLF at end + f"\r\n{REMOTE_URL_TO_TEST}", # CRLF at beginning + f"{REMOTE_URL_TO_TEST}\r\n", # CRLF at end ], ) def test_url_with_nonprintable_characters(self, url_with_nonprintable: str): diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 2534c112486..e1ec1da1aab 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -20,7 +20,6 @@ from requests import ( from galaxy.tool_util.verify.interactor import ValidToolTestDict from galaxy.util import galaxy_root_path -from galaxy.util.unittest_utils import skip_if_github_down from galaxy_test.base import rules_test_data from galaxy_test.base.api_asserts import ( assert_error_code_is, @@ -365,13 +364,16 @@ class TestToolsApi(ApiTestCase, TestsTools): assert "ex2" in option_values @skip_without_tool("test_data_source") - @skip_if_github_down - def test_data_source_ok_request(self): + def test_data_source_ok_request(self, mock_http_server): with self.dataset_populator.test_history() as history_id: + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed", + file_path="test-data/1.bed", + ) payload = self.dataset_populator.run_tool_payload( tool_id="test_data_source", inputs={ - "URL": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed", + "URL": url, "URL_method": "get", "data_type": "bed", }, diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index 7900b6e6e3e..e74dc3eff08 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -12,10 +12,6 @@ from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.util import UNKNOWN from galaxy.util.compression_utils import decompress_bytes_to_directory from galaxy.util.hash_util import md5_hash_file -from galaxy.util.unittest_utils import ( - skip_if_github_down, - skip_if_site_down, -) from galaxy_test.base.constants import ( ONE_TO_SIX_ON_WINDOWS, ONE_TO_SIX_WITH_SPACES, @@ -341,18 +337,23 @@ class TestToolsUpload(ApiTestCase): content = self.dataset_populator.get_history_dataset_content(history_id=history_id, dataset=dataset) assert not content.startswith(">hg17") - @skip_if_github_down - def test_upload_multiple_mixed_success(self, history_id): + def test_upload_multiple_mixed_success(self, history_id, mock_http_server): + url_ok = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed", + file_path="test-data/1.bed", + ) + url_error = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + status=404, + body="Not Found", + ) destination = {"type": "hdas"} targets = [ { "destination": destination, "items": [ - {"src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed"}, - { - "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", - }, + {"src": "url", "url": url_ok}, + {"src": "url", "url": url_error}, ], } ] @@ -371,18 +372,24 @@ class TestToolsUpload(ApiTestCase): assert output0["state"] == "ok" assert output1["state"] == "error" - @skip_if_github_down - def test_fetch_bam_file_from_url_with_extension_set(self, history_id): + def test_fetch_bam_file_from_url_with_extension_set(self, history_id, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + file_path="test-data/1.bam", + ) item = { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam", + "url": url, "ext": "bam", } output = self.dataset_populator.fetch_hda(history_id, item) self.dataset_populator.get_history_dataset_details(history_id, dataset=output, assert_ok=True) - @skip_if_github_down - def test_fetch_html_from_url(self, history_id): + def test_fetch_html_from_url(self, history_id, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/html_file.txt", + file_path="test-data/html_file.txt", + ) destination = {"type": "hdas"} targets = [ { @@ -390,7 +397,7 @@ class TestToolsUpload(ApiTestCase): "items": [ { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/html_file.txt", + "url": url, }, ], } @@ -409,13 +416,18 @@ class TestToolsUpload(ApiTestCase): assert dataset["state"] == "error" assert dataset["name"] == "html_file.txt" - @skip_if_site_down("https://httpstat.us") - def test_abort_fetch_job(self, history_id): + def test_abort_fetch_job(self, history_id, mock_http_server): # This should probably be an integration test that also verifies # that the celery chord is properly canceled. + url = mock_http_server.get_url( + remote_url="https://httpstat.us/200?sleep=10000", + status=200, + body="OK", + sleep_ms=10000, + ) item = { "src": "url", - "url": "https://httpstat.us/200?sleep=10000", + "url": url, "ext": "txt", } destination = {"type": "hdas"} @@ -974,9 +986,13 @@ class TestToolsUpload(ApiTestCase): with pytest.raises(AssertionError): self._upload("https://foo.invalid", assert_ok=False) - @skip_if_site_down("https://usegalaxy.org") - def test_upload_from_404_url(self): - history_id, new_dataset = self._upload("https://usegalaxy.org/bla123", assert_ok=False) + def test_upload_from_404_url(self, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://usegalaxy.org/bla123", + status=404, + body="Not Found", + ) + history_id, new_dataset = self._upload(url, assert_ok=False) dataset_details = self.dataset_populator.get_history_dataset_details( history_id, dataset_id=new_dataset["id"], assert_ok=False ) @@ -984,14 +1000,24 @@ class TestToolsUpload(ApiTestCase): dataset_details["state"] == "error" ), f"expected dataset state to be 'error', but got '{dataset_details['state']}'" - @skip_if_site_down("https://usegalaxy.org") - def test_upload_from_valid_url(self): - history_id, new_dataset = self._upload("https://usegalaxy.org/api/version") + def test_upload_from_valid_url(self, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://usegalaxy.org/api/version", + status=200, + body='{"version_major": "mock"}', + content_type="application/json", + ) + history_id, new_dataset = self._upload(url) self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=True) - @skip_if_site_down("https://usegalaxy.org") - def test_upload_from_valid_url_spaces(self): - history_id, new_dataset = self._upload(" https://usegalaxy.org/api/version ") + def test_upload_from_valid_url_spaces(self, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://usegalaxy.org/api/version", + status=200, + body='{"version_major": "mock"}', + content_type="application/json", + ) + history_id, new_dataset = self._upload(f" {url} ") self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=True) def test_upload_and_validate_invalid(self): diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index fae50aa580f..00fd842e866 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1496,9 +1496,12 @@ steps: other_import_response = self.__import_workflow(workflow_id) self._assert_status_code_is(other_import_response, 403) - @skip_if_github_down - def test_url_import(self): - url = "https://raw.githubusercontent.com/galaxyproject/galaxy/release_19.09/test/base/data/test_workflow_1.ga" + def test_url_import(self, mock_http_server): + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/release_19.09/test/base/data/test_workflow_1.ga", + file_path="lib/galaxy_test/base/data/test_workflow_1.ga", + content_type="application/json", + ) workflow_id = self._post("workflows", data={"archive_source": url}).json()["id"] workflow = self._download_workflow(workflow_id) assert "TestWorkflow1" in workflow["name"] diff --git a/lib/galaxy_test/base/mock_http_server.py b/lib/galaxy_test/base/mock_http_server.py new file mode 100644 index 00000000000..4d9899ffa5b --- /dev/null +++ b/lib/galaxy_test/base/mock_http_server.py @@ -0,0 +1,214 @@ +"""A local mock HTTP server for replacing external test dependencies. + +Replaces httpstat.us, usegalaxy.org, and raw.githubusercontent.com URLs +with a local server. Supports configurable status codes, file serving, +HEAD requests, range requests, and response delays. + +When targeting a remote Galaxy server (GALAXY_TEST_EXTERNAL), falls back to +the real external URLs with automatic skip-if-down behavior. +""" + +import os +import re +import threading +import time +from dataclasses import ( + dataclass, + field, +) +from http.server import ( + BaseHTTPRequestHandler, + HTTPServer, +) +from pathlib import Path +from typing import Optional +from urllib.parse import urlparse + +import pytest + +from galaxy.util.unittest_utils import is_site_up + + +@dataclass +class Route: + status: int = 200 + body: bytes = b"" + headers: dict[str, str] = field(default_factory=dict) + sleep_ms: int = 0 + method: str = "GET" + support_head: bool = False + support_ranges: bool = False + + +class MockHTTPRequestHandler(BaseHTTPRequestHandler): + routes: dict[str, Route] = {} + + def _handle_request(self) -> None: + route = self.routes.get(self.path) + if route is None: + self.send_response(404) + self.end_headers() + self.wfile.write(b"Not Found") + return + + is_head = self.command == "HEAD" + if is_head and not route.support_head: + self.send_response(405) + self.end_headers() + return + if not is_head and route.method != self.command: + self.send_response(405) + self.end_headers() + return + + if route.sleep_ms > 0: + time.sleep(route.sleep_ms / 1000.0) + + body = route.body + + # Handle range requests + range_header = self.headers.get("Range") + if range_header and route.support_ranges: + match = re.match(r"bytes=(\d+)-(\d*)", range_header) + if match: + start = int(match.group(1)) + end = int(match.group(2)) if match.group(2) else len(body) - 1 + total = len(body) + body = body[start : end + 1] + self.send_response(206) + for key, value in route.headers.items(): + self.send_header(key, value) + self.send_header("Content-Length", str(len(body))) + self.send_header("Content-Range", f"bytes {start}-{end}/{total}") + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + if not is_head: + self.wfile.write(body) + return + + self.send_response(route.status) + for key, value in route.headers.items(): + self.send_header(key, value) + if route.support_ranges: + self.send_header("Accept-Ranges", "bytes") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if not is_head: + self.wfile.write(body) + + def do_GET(self) -> None: + self._handle_request() + + def do_HEAD(self) -> None: + self._handle_request() + + def do_POST(self) -> None: + self._handle_request() + + def log_message(self, format, *args) -> None: + pass + + +class MockHttpServer: + """A local mock HTTP server with a per-test URL factory. + + When running locally, registers routes on the mock server and returns local URLs. + When targeting a remote Galaxy (GALAXY_TEST_EXTERNAL), returns the remote URL + and skips the test if the remote site is down. + """ + + def __init__( + self, + base_url: Optional[str], + handler_class: Optional[type[MockHTTPRequestHandler]], + is_remote: bool, + ): + self.base_url = base_url + self._handler_class = handler_class + self.is_remote = is_remote + self._counter = 0 + + def get_url( + self, + *, + remote_url: str, + status: int = 200, + body: str | bytes = "", + file_path: Optional[str] = None, + content_type: str = "application/octet-stream", + sleep_ms: int = 0, + response_headers: Optional[dict[str, str]] = None, + request_method: str = "GET", + support_head: bool = False, + support_ranges: bool = False, + ) -> str: + """Register a mock endpoint and return its URL, or return remote_url with skip-if-down. + + Args: + remote_url: The real URL to use when targeting a remote Galaxy server. + status: HTTP status code to return. + body: Response body (string or bytes). Ignored if file_path is set. + file_path: Path to a file whose contents will be served as the response body. + content_type: Content-Type header value. + sleep_ms: Delay in milliseconds before responding. + response_headers: Additional response headers. + request_method: HTTP method to match (GET, POST, etc.). + support_head: Whether to also respond to HEAD requests. + support_ranges: Whether to support Range request headers (HTTP 206). + """ + if self.is_remote: + if not is_site_up(remote_url): + pytest.skip(f"Remote site {remote_url} is down") + return remote_url + + assert self._handler_class is not None + self._counter += 1 + # Include filename in path so tools can sniff file extensions from the URL + if file_path is not None: + filename = os.path.basename(file_path) + else: + filename = os.path.basename(urlparse(remote_url).path) + path = f"/mock/{self._counter}/{filename}" if filename else f"/mock/{self._counter}" + + if file_path is not None: + file_path_obj = Path(file_path) + if not file_path_obj.is_absolute(): + # Resolve relative paths from the Galaxy project root + galaxy_root = Path(__file__).resolve().parents[3] + file_path_obj = galaxy_root / file_path_obj + encoded_body = file_path_obj.read_bytes() + elif isinstance(body, str): + encoded_body = body.encode() + else: + encoded_body = body + + headers = {"Content-Type": content_type} + if response_headers: + headers.update(response_headers) + self._handler_class.routes[path] = Route( + status=status, + body=encoded_body, + headers=headers, + sleep_ms=sleep_ms, + method=request_method, + support_head=support_head, + support_ranges=support_ranges, + ) + return f"{self.base_url}{path}" + + +def start_mock_http_server(host: str = "127.0.0.1", port: int = 0) -> tuple[HTTPServer, str]: + """Start a mock HTTP server in a daemon thread. + + Args: + host: Address to bind to. Defaults to 127.0.0.1. + port: Port to bind to. 0 means OS-assigned. + + Returns: + Tuple of (server, base_url). + """ + server = HTTPServer((host, port), MockHTTPRequestHandler) + actual_port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, f"http://{host}:{actual_port}" diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg index c898e71d5b6..8bd63f4718e 100644 --- a/packages/app/setup.cfg +++ b/packages/app/setup.cfg @@ -85,6 +85,7 @@ python_requires = >=3.10 [options.extras_require] test = + galaxy-test-base mock-ssh-server pkce pykwalify diff --git a/packages/files/setup.cfg b/packages/files/setup.cfg index 6cfdcff63ac..885a5d2e38c 100644 --- a/packages/files/setup.cfg +++ b/packages/files/setup.cfg @@ -42,6 +42,7 @@ python_requires = >=3.10 [options.extras_require] test = + galaxy-test-base pytest gcsfs s3fs>=2023.1.0 diff --git a/packages/packages_by_dep_dag.txt b/packages/packages_by_dep_dag.txt index f592240bbcb..c7b058853a2 100644 --- a/packages/packages_by_dep_dag.txt +++ b/packages/packages_by_dep_dag.txt @@ -2,7 +2,6 @@ tool_util_models util config -files job_metrics navigation objectstore @@ -10,10 +9,13 @@ schema tool_shed_schema tool_util +test_base + +files + data selenium tours -test_base auth job_execution diff --git a/test-data/rocrate-test.zip b/test-data/rocrate-test.zip new file mode 100644 index 00000000000..a379a8f28ca Binary files /dev/null and b/test-data/rocrate-test.zip differ diff --git a/test/functional/tools/parameters/gx_hidden_data.xml b/test/functional/tools/parameters/gx_hidden_data.xml new file mode 100644 index 00000000000..ed89987bda1 --- /dev/null +++ b/test/functional/tools/parameters/gx_hidden_data.xml @@ -0,0 +1,28 @@ + + > '$output' +#else +echo 'NO_DATA' >> '$output' +#end if + ]]> + + + + + + + + + + + + + + + + + + + + diff --git a/test/unit/app/managers/test_markdown_export.py b/test/unit/app/managers/test_markdown_export.py index 4f512a831ec..d9513f5182a 100644 --- a/test/unit/app/managers/test_markdown_export.py +++ b/test/unit/app/managers/test_markdown_export.py @@ -280,7 +280,7 @@ invocation_time(invocation_id=1) invocation = self._new_invocation() self.app.workflow_manager.get_invocation.side_effect = [invocation, invocation] result = self._to_basic(example) - expectedtime = invocation.create_time.strftime("%Y-%m-%d, %H:%M:%S") + expectedtime = invocation.create_time.strftime("%Y-%m-%d, %H:%M:%S UTC") assert f"\n {expectedtime}" in result def test_job_parameters(self): @@ -419,9 +419,7 @@ invocation_time(invocation_id=1) _, result, extra_data = self._ready_export(example) assert "invocations" in extra_data assert "create_time" in extra_data["invocations"]["be8be0fd2ce547f6"] - assert extra_data["invocations"]["be8be0fd2ce547f6"]["create_time"] == invocation.create_time.strftime( - "%Y-%m-%d, %H:%M:%S" - ) + assert extra_data["invocations"]["be8be0fd2ce547f6"]["create_time"] == invocation.create_time.isoformat() def test_export_replaces_embedded_history_dataset_type(self): hda = self._new_hda() diff --git a/test/unit/app/tools/conftest.py b/test/unit/app/tools/conftest.py index 4a63e74d85c..a2d7eff5e6d 100644 --- a/test/unit/app/tools/conftest.py +++ b/test/unit/app/tools/conftest.py @@ -7,6 +7,20 @@ from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping from galaxy.tool_shed.cache import ToolShedRepositoryCache from galaxy.tool_util.toolbox.base import ToolConfRepository +from galaxy_test.base.mock_http_server import ( + MockHTTPRequestHandler, + MockHttpServer, + start_mock_http_server, +) + + +@pytest.fixture(scope="session") +def mock_http_server(): + server, base_url = start_mock_http_server() + try: + yield MockHttpServer(base_url=base_url, handler_class=MockHTTPRequestHandler, is_remote=False) + finally: + server.shutdown() @pytest.fixture diff --git a/test/unit/app/tools/test_column_parameters.py b/test/unit/app/tools/test_column_parameters.py index 45d6505ec49..6777d5be382 100644 --- a/test/unit/app/tools/test_column_parameters.py +++ b/test/unit/app/tools/test_column_parameters.py @@ -57,6 +57,11 @@ class TestDataColumnParameter(BaseParameterTestCase): self.other_attributes = "value='c2'" assert "2" == self.param.get_initial_value(self.trans, {"input_tsv": self.build_ready_hda()}) + def test_get_options_hdca_with_usecolnames(self): + self.other_attributes = 'use_header_names="true"' + options = self.param.get_options(self.trans, {"input_tsv": self.build_ready_hdca()}) + assert len(options) > 0 + def setUp(self): super().setUp() self.test_history = model.History() @@ -89,6 +94,16 @@ class TestDataColumnParameter(BaseParameterTestCase): ready_hda.state = "ok" return ready_hda + def build_ready_hdca(self): + hda = self.build_ready_hda() + session = self.app.model.context + c1 = model.DatasetCollection(collection_type="list") + model.DatasetCollectionElement(collection=c1, element=hda, element_identifier="sample1", element_index=0) + hdca = model.HistoryDatasetCollectionAssociation(collection=c1, name="test_collection") + session.add(hdca) + session.commit() + return hdca + @property def param(self): if not self._param: diff --git a/test/unit/app/tools/test_data_fetch.py b/test/unit/app/tools/test_data_fetch.py index 667e5e7ebbb..9ce94d6d36a 100644 --- a/test/unit/app/tools/test_data_fetch.py +++ b/test/unit/app/tools/test_data_fetch.py @@ -10,7 +10,6 @@ from typing import Optional import pytest from galaxy.tools.data_fetch import main -from galaxy.util.unittest_utils import skip_if_github_down B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8") URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}" @@ -63,9 +62,12 @@ def test_simple_path_get(hash_value: str, error_message: Optional[str]): assert "error_message" not in hda_result -@skip_if_github_down -def test_simple_uri_get(): - with _execute_context() as execute_context: +def test_simple_uri_get(mock_http_server): + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed", + file_path="test-data/1.bed", + ) + with _execute_context(allow_localhost=True) as execute_context: request = { "targets": [ { @@ -75,7 +77,7 @@ def test_simple_uri_get(): "elements": [ { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed", + "url": url, } ], } @@ -212,9 +214,13 @@ def test_incorrect_sha1(): ) -@skip_if_github_down -def test_deferred_uri_get(): - with _execute_context() as execute_context: +def test_deferred_uri_get(mock_http_server): + url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + status=404, + body="Not Found", + ) + with _execute_context(allow_localhost=True) as execute_context: request = { "targets": [ { @@ -224,7 +230,7 @@ def test_deferred_uri_get(): "elements": [ { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + "url": url, "deferred": True, } ], @@ -262,9 +268,13 @@ def test_simple_list_path_get(): assert destination["object_id"] == 76 -@skip_if_github_down -def test_hdas_single_url_error(): - with _execute_context() as execute_context: +def test_hdas_single_url_error(mock_http_server): + url_12_bed = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + status=404, + body="Not Found", + ) + with _execute_context(allow_localhost=True) as execute_context: job_directory = execute_context.job_directory example_path = os.path.join(job_directory, "example_file") with open(example_path, "w") as f: @@ -279,7 +289,7 @@ def test_hdas_single_url_error(): {"src": "path", "path": example_path}, { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + "url": url_12_bed, }, ], } @@ -293,14 +303,16 @@ def test_hdas_single_url_error(): assert "error_message" not in elements[0] assert "error_message" in elements[1] error = elements[1]["error_message"] - assert ( - "Failed to fetch url https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed" in error - ) + assert f"Failed to fetch url {url_12_bed}" in error -@skip_if_github_down -def test_hdca_collection_element_failed(): - with _execute_context() as execute_context: +def test_hdca_collection_element_failed(mock_http_server): + url_12_bed = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + status=404, + body="Not Found", + ) + with _execute_context(allow_localhost=True) as execute_context: job_directory = execute_context.job_directory example_path = os.path.join(job_directory, "example_file") with open(example_path, "w") as f: @@ -315,7 +327,7 @@ def test_hdca_collection_element_failed(): {"src": "path", "path": example_path}, { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + "url": url_12_bed, }, ], } @@ -325,14 +337,16 @@ def test_hdca_collection_element_failed(): output = _unnamed_output(execute_context) assert "error_message" in output error = output["error_message"] - assert ( - "Failed to fetch url https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed" in error - ) + assert f"Failed to fetch url {url_12_bed}" in error -@skip_if_github_down -def test_hdca_allow_failed_collections(): - with _execute_context() as execute_context: +def test_hdca_allow_failed_collections(mock_http_server): + url_12_bed = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + status=404, + body="Not Found", + ) + with _execute_context(allow_localhost=True) as execute_context: job_directory = execute_context.job_directory example_path = os.path.join(job_directory, "example_file") with open(example_path, "w") as f: @@ -348,7 +362,7 @@ def test_hdca_allow_failed_collections(): {"src": "path", "path": example_path}, { "src": "url", - "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed", + "url": url_12_bed, }, ], } @@ -363,9 +377,7 @@ def test_hdca_allow_failed_collections(): assert "error_message" not in elements[0] assert "error_message" in elements[1] error = elements[1]["error_message"] - assert ( - "Failed to fetch url https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed" in error - ) + assert f"Failed to fetch url {url_12_bed}" in error def test_hdca_failed_expansion(): @@ -397,9 +409,32 @@ def test_hdca_failed_expansion(): @contextmanager -def _execute_context(): +def _execute_context(allow_localhost=False): job_directory = mkdtemp() try: + if allow_localhost: + file_sources_path = os.path.join(job_directory, "file_sources.json") + with open(file_sources_path, "w") as f: + json.dump( + { + "file_sources": [ + {"type": "http", "id": "stock_http"}, + {"type": "base64", "id": "stock_base64"}, + ], + "config": { + "symlink_allowlist": [], + "fetch_url_allowlist": ["127.0.0.0/24"], + "library_import_dir": None, + "user_library_import_dir": None, + "ftp_upload_dir": None, + "ftp_upload_purge": True, + "tmp_dir": None, + "webdav_use_temp_files": None, + "listings_expiry_time": None, + }, + }, + f, + ) # temporarily set tempdir to non-existing location # to make sure all intermediate files are created in the working # directory diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 76961e6adee..75028593421 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -218,6 +218,61 @@ class TestMappings(BaseModelTestCase): ] assert c4.dataset_elements == [dce1, dce2] + def test_nested_collection_attributes_duplicate_child_collection_id(self): + """Regression test for CardinalityViolation when multiple parent elements + reference the same child collection (duplicate child_collection_id). + + On PostgreSQL the ARRAY walk optimisation uses correlated scalar + subqueries to navigate from leaf elements back to their ancestors via + child_collection_id. When two parent DatasetCollectionElements share + the same child_collection_id those subqueries return more than one row, + causing 'more than one row returned by a subquery used as an expression'. + """ + u = model.User(email=random_email(), password="password") + h1 = model.History(name="History 1", user=u) + + # Build a paired collection with two datasets. + pair = model.DatasetCollection(collection_type="paired") + forward = model.HistoryDatasetAssociation( + extension="txt", history=h1, create_dataset=True, sa_session=self.model.session + ) + reverse = model.HistoryDatasetAssociation( + extension="bam", history=h1, create_dataset=True, sa_session=self.model.session + ) + model.DatasetCollectionElement(collection=pair, element=forward, element_identifier="forward", element_index=0) + model.DatasetCollectionElement(collection=pair, element=reverse, element_identifier="reverse", element_index=1) + + # Create a list:paired collection where the *same* paired collection + # is referenced by two parent elements (duplicate child_collection_id). + list_pair = model.DatasetCollection(collection_type="list:paired") + model.DatasetCollectionElement( + collection=list_pair, element=pair, element_identifier="sample1", element_index=0 + ) + model.DatasetCollectionElement( + collection=list_pair, element=pair, element_identifier="sample2", element_index=1 + ) + + self.persist(u, h1, forward, reverse, pair, list_pair, commit=True, expunge=False) + self.model.session.flush() + + # All query paths that use _build_nested_collection_attributes_stmt + # must tolerate the duplicate without raising CardinalityViolation. + stmt = list_pair._build_nested_collection_attributes_stmt( + element_attributes=("element_identifier",), + hda_attributes=("extension",), + dataset_attributes=("state",), + ) + result = self.model.session.execute(stmt).all() + # The two leaf datasets should appear (potentially duplicated because + # the same child collection is reachable via two parents). + extensions_in_result = {r.extension for r in result} + assert extensions_in_result == {"txt", "bam"} + + # dataset_states_and_extensions_summary must not raise either. + summary = list_pair.dataset_states_and_extensions_summary + assert "txt" in summary.extensions + assert "bam" in summary.extensions + def test_history_audit(self): u = model.User(email=random_email(), password="password") h1 = model.History(name="HistoryAuditHistory", user=u) diff --git a/test/unit/files/conftest.py b/test/unit/files/conftest.py new file mode 100644 index 00000000000..41286de8f5a --- /dev/null +++ b/test/unit/files/conftest.py @@ -0,0 +1,16 @@ +import pytest + +from galaxy_test.base.mock_http_server import ( + MockHTTPRequestHandler, + MockHttpServer, + start_mock_http_server, +) + + +@pytest.fixture(scope="session") +def mock_http_server(): + server, base_url = start_mock_http_server() + try: + yield MockHttpServer(base_url=base_url, handler_class=MockHTTPRequestHandler, is_remote=False) + finally: + server.shutdown() diff --git a/test/unit/files/test_remotezip.py b/test/unit/files/test_remotezip.py index 7efc0ee4c33..3c5579560e0 100644 --- a/test/unit/files/test_remotezip.py +++ b/test/unit/files/test_remotezip.py @@ -1,9 +1,10 @@ +import ipaddress import os import pytest from galaxy.exceptions import AdminRequiredException -from galaxy.util.unittest_utils import skip_if_github_down +from galaxy.files.models import FileSourcePluginsConfig from ._util import ( assert_realizes_contains, configured_file_sources, @@ -13,19 +14,28 @@ from ._util import ( SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "remotezip_file_sources_conf.yml") +LOCALHOST_ALLOWLIST = FileSourcePluginsConfig( + fetch_url_allowlist=[ipaddress.ip_network("127.0.0.0/24")], +) -@skip_if_github_down -def test_file_source(): - remote_zip_url = "https://raw.githubusercontent.com/davelopez/ro-crate-zip-explorer/refs/heads/main/tests/test-data/rocrate-test.zip" +def test_file_source(mock_http_server): + zip_url = mock_http_server.get_url( + remote_url="https://raw.githubusercontent.com/davelopez/ro-crate-zip-explorer/refs/heads/main/tests/test-data/rocrate-test.zip", + file_path="test-data/rocrate-test.zip", + content_type="application/zip", + support_ranges=True, + support_head=True, + ) header_offset = 15875 compression_method = 8 compress_size = 582 expected_contents = "class: GalaxyWorkflow" - test_url = f"zip://extract?source={remote_zip_url}&header_offset={header_offset}&compress_size={compress_size}&compression_method={compression_method}" + test_url = f"zip://extract?source={zip_url}&header_offset={header_offset}&compress_size={compress_size}&compression_method={compression_method}" user_context = user_context_fixture() - file_sources = configured_file_sources(FILE_SOURCES_CONF) + file_sources_config = LOCALHOST_ALLOWLIST if not mock_http_server.is_remote else None + file_sources = configured_file_sources(FILE_SOURCES_CONF, file_sources_config=file_sources_config) file_source_pair = file_sources.get_file_source_path(test_url) assert file_source_pair.path == test_url diff --git a/test/unit/tool_util/parameter_specification.yml b/test/unit/tool_util/parameter_specification.yml index 15aa958b114..62889281944 100644 --- a/test/unit/tool_util/parameter_specification.yml +++ b/test/unit/tool_util/parameter_specification.yml @@ -1368,11 +1368,11 @@ gx_data: gx_data_optional: - request_valid: + request_valid: &data_optional_request_valid - parameter: {src: hda, id: abcdabcd} - parameter: null - {} - request_invalid: + request_invalid: &data_optional_request_invalid - parameter: {src: hda, id: 5} - parameter: {src: hdca, id: abcdabcd} - parameter: {src: fooda, id: abcdabcd} @@ -1381,11 +1381,11 @@ gx_data_optional: - parameter: true - parameter: 5 - parameter: "5" - request_internal_valid: + request_internal_valid: &data_optional_request_internal_valid - parameter: {src: hda, id: 5} - parameter: null - {} - request_internal_invalid: + request_internal_invalid: &data_optional_request_internal_invalid - parameter: {src: hda, id: abcdabcd} - parameter: {src: hdca, id: abcdabcd} - parameter: {src: fooda, id: abcdabcd} @@ -1394,10 +1394,10 @@ gx_data_optional: - parameter: true - parameter: 5 - parameter: "5" - job_internal_valid: + job_internal_valid: &data_optional_job_internal_valid - parameter: null - parameter: {src: hda, id: 5} - job_internal_invalid: + job_internal_invalid: &data_optional_job_internal_invalid - {} job_runtime_valid: - parameter: {class: File, basename: "f.txt", location: "step_input://0", @@ -1407,27 +1407,41 @@ gx_data_optional: - parameter: {src: hda, id: 7} - parameter: {class: File} - {} - workflow_step_valid: + workflow_step_valid: &data_optional_workflow_step_valid - {} - workflow_step_invalid: + workflow_step_invalid: &data_optional_workflow_step_invalid - {src: hda, id: abcdabcd} - {src: hda, id: 7} - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} - parameter: {__class__: 'ConnectedValue'} - workflow_step_linked_valid: + workflow_step_linked_valid: &data_optional_workflow_step_linked_valid - {} - parameter: {__class__: 'ConnectedValue'} - workflow_step_linked_invalid: + workflow_step_linked_invalid: &data_optional_workflow_step_linked_invalid - {src: hda, id: abcdabcd} - {src: hda, id: 7} - parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]} - parameter: {__class__: 'ConnectedValueX'} - test_case_xml_valid: + test_case_xml_valid: &data_optional_test_case_xml_valid - {} - parameter: {class: "File", path: "1.bed"} - test_case_xml_invalid: + test_case_xml_invalid: &data_optional_test_case_xml_invalid - parameter: {class: "NotAFile", path: "1.bed"} +gx_hidden_data: + request_valid: *data_optional_request_valid + request_invalid: *data_optional_request_invalid + request_internal_valid: *data_optional_request_internal_valid + request_internal_invalid: *data_optional_request_internal_invalid + job_internal_valid: *data_optional_job_internal_valid + job_internal_invalid: *data_optional_job_internal_invalid + workflow_step_valid: *data_optional_workflow_step_valid + workflow_step_invalid: *data_optional_workflow_step_invalid + workflow_step_linked_valid: *data_optional_workflow_step_linked_valid + workflow_step_linked_invalid: *data_optional_workflow_step_linked_invalid + test_case_xml_valid: *data_optional_test_case_xml_valid + test_case_xml_invalid: *data_optional_test_case_xml_invalid + gx_data_multiple: request_valid: - parameter: {src: hda, id: abcdabcd}