opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-29 17:42:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_26.0' into dev

Browse Source
This commit is contained in:
mvdbeek
2026-03-17 16:05:21 +01:00
parent 73105a1acd dd290a7c36
commit 118f934499
38 changed files with 959 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/package.json
+1
View File
@@ -199,6 +199,7 @@
"prettier": "^3.6.2",
"sass": "^1.94.2",
"store": "^2.0.12",
"timezone-mock": "^1.3.6",
"typescript": "^5.7.3",
"vite": "^7.2.4",
"vite-tsconfig-paths": "^5.1.4",
client/pnpm-lock.yaml
Generated
+8
View File
@@ -458,6 +458,9 @@ importers:
store:
specifier: ^2.0.12
version: 2.0.12
timezone-mock:
specifier: ^1.3.6
version: 1.4.2
typescript:
specifier: ^5.7.3
version: 5.8.3
@@ -3965,6 +3968,9 @@ packages:
resolution: {integrity: sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==}
engines: {node: '>=0.6.0'}
timezone-mock@1.4.2:
resolution: {integrity: sha512-RgPQkSrARNHBnchuFpjrYf97uWgEaseCXK2asSeGcgUzRUO3cBR9Zu0uO1z2Eyi/TQsQtHIvQLi0br30COHWUA==}
tinybench@2.9.0:
resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
@@ -8313,6 +8319,8 @@ snapshots:
dependencies:
setimmediate: 1.0.5
timezone-mock@1.4.2: {}
tinybench@2.9.0: {}
tinyexec@0.3.2: {}
client/src/components/ActivityBar/ActivitySettings.vue
+24 -22
View File
@@ -110,28 +110,30 @@ function executeActivity(activity: Activity) {
@click.stop="onRemove(activity)">
<FontAwesomeIcon :icon="faTrash" fa-fw />
</GButton>
<GButton
v-if="activity.visible"
tooltip
size="small"
transparent
icon-only
color="blue"
title="Hide in Activity Bar"
@click.stop="onFavorite(activity)">
<FontAwesomeIcon :icon="faStar" fa-fw />
</GButton>
<GButton
v-else
tooltip
transparent
icon-only
color="blue"
size="small"
title="Show in Activity Bar"
@click.stop="onFavorite(activity)">
<FontAwesomeIcon :icon="faStarRegular" fa-fw />
</GButton>
<template v-if="activity.optional">
<GButton
v-if="activity.visible"
tooltip
size="small"
transparent
icon-only
color="blue"
title="Hide in Activity Bar"
@click.stop="onFavorite(activity)">
<FontAwesomeIcon :icon="faStar" fa-fw />
</GButton>
<GButton
v-else
tooltip
transparent
icon-only
color="blue"
size="small"
title="Show in Activity Bar"
@click.stop="onFavorite(activity)">
<FontAwesomeIcon :icon="faStarRegular" fa-fw />
</GButton>
</template>
</div>
</span>
</div>
client/src/components/UtcDate.vue
+5 -3
View File
@@ -1,7 +1,9 @@
<script setup lang="ts">
import { format, formatDistanceToNow, parseISO } from "date-fns";
import { format, formatDistanceToNow } from "date-fns";
import { computed } from "vue";
import { galaxyTimeToDate, localizeUTCPretty } from "@/utils/dates";
interface UtcDateProps {
date: string;
mode?: "date" | "elapsed" | "pretty" | "timeonly";
@@ -11,10 +13,10 @@ const props = withDefaults(defineProps<UtcDateProps>(), {
mode: "date",
});
const parsedDate = computed(() => parseISO(`${props.date}Z`));
const parsedDate = computed(() => galaxyTimeToDate(props.date));
const elapsedTime = computed(() => formatDistanceToNow(parsedDate.value, { addSuffix: true }));
const fullISO = computed(() => parsedDate.value.toISOString());
const pretty = computed(() => format(parsedDate.value, "eeee MMM do H:mm:ss yyyy zz"));
const pretty = computed(() => localizeUTCPretty(parsedDate.value));
const timeonly = computed(() => format(parsedDate.value, "H:mm:ss zz"));
</script>
client/src/utils/dates.test.ts
+52
View File
@@ -0,0 +1,52 @@
import MockDate from "timezone-mock";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { formatGalaxyPrettyDateString, galaxyTimeToDate, localizeUTCPretty } from "./dates";
describe("dates.ts", () => {
beforeEach(() => {
MockDate.register("Etc/GMT+4");
});
afterEach(() => {
MockDate.unregister();
});
describe("galaxyTimeToDate", () => {
it("should convert valid galaxyTime string to Date object", () => {
const galaxyTime = "2023-10-01T12:00:00";
const date = galaxyTimeToDate(galaxyTime);
expect(date).toBeInstanceOf(Date);
expect(date.toISOString()).toBe("2023-10-01T12:00:00.000Z");
});
it("should append Z if missing and parse correctly", () => {
const galaxyTime = "2023-10-01T12:00:00";
const date = galaxyTimeToDate(galaxyTime);
expect(date.toISOString()).toBe("2023-10-01T12:00:00.000Z");
});
it("should throw an error for invalid galaxyTime string", () => {
const invalidGalaxyTime = "invalid-date-string";
expect(() => galaxyTimeToDate(invalidGalaxyTime)).toThrow(
`Invalid galaxyTime string: ${invalidGalaxyTime}`,
);
});
});
describe("localizeUTCPretty", () => {
it("should format Date object into human-readable string", () => {
const date = new Date("2023-10-01T12:00:00Z");
const formatted = localizeUTCPretty(date);
expect(formatted).toBe("Sunday Oct 1st 8:00:00 2023 GMT-4");
});
});
describe("formatGalaxyPrettyDateString", () => {
it("should convert galaxyTime string to formatted date string", () => {
const galaxyTime = "2023-10-01T12:00:00";
const formatted = formatGalaxyPrettyDateString(galaxyTime);
expect(formatted).toBe("Sunday Oct 1st 8:00:00 2023 GMT-4");
});
});
});
client/src/utils/dates.ts
+39
View File
@@ -0,0 +1,39 @@
import { format, parseISO } from "date-fns";
/**
* Converts a Galaxy time string to a Date object.
* @param {string} galaxyTime - The Galaxy time string in ISO format.
* @returns {Date} The parsed Date object.
*/
export function galaxyTimeToDate(galaxyTime: string): Date {
// Galaxy doesn't include Zulu time zone designator, but it's always UTC
// so we need to add it to parse the string correctly in JavaScript.
let time = galaxyTime;
if (!time.endsWith("Z")) {
time += "Z";
}
const date = parseISO(time);
if (isNaN(date.getTime())) {
throw new Error(`Invalid galaxyTime string: ${galaxyTime}`);
}
return date;
}
/**
* Formats a UTC Date object into a human-readable string, localized to the user's time zone.
* @param {Date} utcDate - The UTC Date object.
* @returns {string} The formatted date string.
*/
export function localizeUTCPretty(utcDate: Date): string {
return format(utcDate, "eeee MMM do H:mm:ss yyyy zz");
}
/**
* Converts a Galaxy time string to a human-readable formatted date string, localized to the user's time zone.
* @param {string} galaxyTime - The Galaxy time string in ISO format.
* @returns {string} The formatted date string.
*/
export function formatGalaxyPrettyDateString(galaxyTime: string): string {
const date = galaxyTimeToDate(galaxyTime);
return localizeUTCPretty(date);
}
lib/galaxy/managers/markdown_util.py
+6 -5
View File
@@ -620,9 +620,7 @@ class ReadyForExportMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHand
pass
def handle_invocation_time(self, line, invocation):
self.ensure_rendering_data_for("invocations", invocation)["create_time"] = _database_time_to_str(
invocation.create_time
)
self.ensure_rendering_data_for("invocations", invocation)["create_time"] = invocation.create_time.isoformat()
def handle_invocation_inputs(self, line, invocation):
pass
@@ -677,6 +675,9 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler):
def __init__(self, trans):
self.trans = trans
def _format_printable_time(self, time):
return time.strftime("%Y-%m-%d, %H:%M:%S UTC")
def handle_dataset_display(self, line, hda):
name = hda.name or ""
markdown = "---\n"
@@ -852,7 +853,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler):
return (content, True)
def handle_generate_time(self, line, generate_time):
content = literal_via_fence(generate_time.isoformat())
content = literal_via_fence(self._format_printable_time(generate_time))
return (content, True)
def handle_instance_access_link(self, line, url):
@@ -884,7 +885,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler):
return (content, True)
def handle_invocation_time(self, line, invocation):
content = literal_via_fence(_database_time_to_str(invocation.create_time))
content = literal_via_fence(self._format_printable_time(invocation.create_time))
return (content, True)
def handle_invocation_inputs(self, line, invocation):
lib/galaxy/model/__init__.py
+203 -38
View File
@@ -59,6 +59,7 @@ from social_core.storage import (
from sqlalchemy import (
alias,
and_,
any_,
asc,
BigInteger,
bindparam,
@@ -158,6 +159,7 @@ from galaxy.model.custom_types import (
UUIDType,
)
from galaxy.model.database_object_names import NAMING_CONVENTION
from galaxy.model.database_utils import supports_skip_locked as _check_supports_skip_locked
from galaxy.model.item_attrs import (
get_item_annotation_str,
UsesAnnotations,
@@ -4801,6 +4803,8 @@ class Dataset(Base, StorableObject, Serializable):
dialect_name = session.bind.dialect.name
if dialect_name in ("postgresql", "sqlite"):
if dialect_name == "postgresql" and supports_skip_locked is None:
supports_skip_locked = _check_supports_skip_locked(session.get_bind())
self._touch_collection_update_time_cte(session, supports_skip_locked)
else:
# Fallback for other databases
@@ -7066,49 +7070,169 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable):
dc = alias(DatasetCollection.__table__)
dce = alias(DatasetCollectionElement.__table__)
depth_collection_type = dataset_collection.collection_type
order_by_columns = [dce.c.element_index]
nesting_level = 0
def attribute_columns(column_collection, attributes, nesting_level=None):
label_fragment = f"_{nesting_level}" if nesting_level is not None else ""
return [getattr(column_collection, a).label(f"{a}{label_fragment}") for a in attributes]
q = (
select(
*attribute_columns(dce.c, element_attributes, nesting_level),
*attribute_columns(dc.c, collection_attributes, nesting_level),
# On PostgreSQL, joining dataset_collection_element to itself causes
# the planner to choose catastrophically slow hash/merge joins: it
# overestimates cardinality and scans the entire table at each level.
# The ARRAY(subquery) pattern avoids all DCE-to-DCE joins by
# evaluating each nesting level into a concrete array before the next
# lookup, forcing index scans throughout.
#
# Per-level element attributes and hierarchical ordering are
# provided via correlated scalar subqueries that walk UP from each
# leaf DCE to its ancestors via the indexed child_collection_id
# column. Each subquery returns exactly one row via an index lookup.
#
# collection_attributes (e.g. populated_state) must stay on the
# outerjoin path: the ARRAY walk only visits branches that contain
# leaf DCEs, so empty sub-collections — common when conditional
# workflow steps are skipped — would be invisible and their
# populated_state never checked.
try:
session = object_session(self)
except Exception:
session = None
is_postgres = session is not None and session.bind and session.bind.dialect.name == "postgresql"
use_array_walk = is_postgres and ":" in depth_collection_type and not collection_attributes
if use_array_walk:
# Build nested ARRAY(subquery) expressions to walk the tree
# without any DCE-to-DCE joins.
dce_table = DatasetCollectionElement.__table__
inner_dce = alias(dce_table)
child_ids_subq = select(inner_dce.c.child_collection_id).where(
inner_dce.c.dataset_collection_id == dataset_collection.id
)
.select_from(dce, dc)
.join(dce, dce.c.dataset_collection_id == dc.c.id)
.filter(dc.c.id == dataset_collection.id)
)
while ":" in depth_collection_type:
nesting_level += 1
inner_dce = alias(DatasetCollectionElement.__table__)
inner_dc = alias(DatasetCollection.__table__)
order_by_columns.append(inner_dce.c.element_index)
q = q.outerjoin(inner_dce, inner_dce.c.dataset_collection_id == dce.c.child_collection_id)
if collection_attributes:
q = q.outerjoin(inner_dc, inner_dc.c.id == dce.c.child_collection_id)
q = q.add_columns(
*attribute_columns(inner_dc.c, collection_attributes, nesting_level),
n_intermediates = depth_collection_type.count(":")
for _ in range(n_intermediates - 1):
next_dce = alias(dce_table)
child_ids_subq = select(next_dce.c.child_collection_id).where(
next_dce.c.dataset_collection_id == any_(func.array(child_ids_subq.scalar_subquery()))
)
q = q.add_columns(*attribute_columns(inner_dce.c, element_attributes, nesting_level))
dce = inner_dce
dc = inner_dc
depth_collection_type = depth_collection_type.split(":", 1)[1]
leaf_dce = alias(dce_table)
q = (
select()
.select_from(leaf_dce)
.where(leaf_dce.c.dataset_collection_id == any_(func.array(child_ids_subq.scalar_subquery())))
)
n_levels = n_intermediates + 1
if (
# Build navigation chain of collection ID expressions from leaf
# to root. coll_ids[0] = leaf's collection, coll_ids[k] = k
# steps up from the leaf. Each entry after the first is a
# correlated scalar subquery that walks one level up via the
# indexed child_collection_id column.
coll_ids: list = [leaf_dce.c.dataset_collection_id]
for _i in range(n_intermediates):
nav = alias(dce_table)
coll_ids.append(
select(nav.c.dataset_collection_id)
.where(nav.c.child_collection_id == coll_ids[-1])
.correlate(leaf_dce)
.limit(1)
.scalar_subquery()
)
# Add element attribute columns for each level (root first,
# leaf last) to match the column ordering of the outerjoin path.
for level in range(n_levels):
steps_up = n_levels - 1 - level
if steps_up == 0:
# Leaf level: direct column access
q = q.add_columns(*[getattr(leaf_dce.c, a).label(f"{a}_{level}") for a in element_attributes])
else:
# Ancestor level: correlated subquery
for attr in element_attributes:
nav = alias(dce_table)
q = q.add_columns(
select(getattr(nav.c, attr))
.where(nav.c.child_collection_id == coll_ids[steps_up - 1])
.correlate(leaf_dce)
.limit(1)
.scalar_subquery()
.label(f"{attr}_{level}")
)
hda_join_col = leaf_dce.c.hda_id
dce_id_col = leaf_dce.c.id
# Build hierarchical ORDER BY from root level to leaf level.
order_by_columns: list[ColumnElement] = []
for level in range(n_levels):
steps_up = n_levels - 1 - level
if steps_up == 0:
order_by_columns.append(leaf_dce.c.element_index)
else:
nav = alias(dce_table)
order_by_columns.append(
select(nav.c.element_index)
.where(nav.c.child_collection_id == coll_ids[steps_up - 1])
.correlate(leaf_dce)
.limit(1)
.scalar_subquery()
)
elif ":" not in depth_collection_type:
# Single-level collection: simple join, no nesting issues.
q = (
select(
*attribute_columns(dce.c, element_attributes, nesting_level),
*attribute_columns(dc.c, collection_attributes, nesting_level),
)
.select_from(dce, dc)
.join(dce, dce.c.dataset_collection_id == dc.c.id)
.filter(dc.c.id == dataset_collection.id)
)
hda_join_col = dce.c.hda_id
dce_id_col = dce.c.id
order_by_columns = [dce.c.element_index]
else:
# Nested collection on SQLite, or collection_attributes on
# PostgreSQL: use the original outerjoin approach.
order_by_columns = [dce.c.element_index]
q = (
select(
*attribute_columns(dce.c, element_attributes, nesting_level),
*attribute_columns(dc.c, collection_attributes, nesting_level),
)
.select_from(dce, dc)
.join(dce, dce.c.dataset_collection_id == dc.c.id)
.filter(dc.c.id == dataset_collection.id)
)
while ":" in depth_collection_type:
nesting_level += 1
inner_dce = alias(DatasetCollectionElement.__table__)
inner_dc = alias(DatasetCollection.__table__)
order_by_columns.append(inner_dce.c.element_index)
q = q.outerjoin(inner_dce, inner_dce.c.dataset_collection_id == dce.c.child_collection_id)
if collection_attributes:
q = q.outerjoin(inner_dc, inner_dc.c.id == dce.c.child_collection_id)
q = q.add_columns(
*attribute_columns(inner_dc.c, collection_attributes, nesting_level),
)
q = q.add_columns(*attribute_columns(inner_dce.c, element_attributes, nesting_level))
dce = inner_dce
dc = inner_dc
depth_collection_type = depth_collection_type.split(":", 1)[1]
hda_join_col = dce.c.hda_id
dce_id_col = dce.c.id
needs_hda_join = bool(
hda_attributes
or dataset_attributes
or dataset_permission_attributes
or return_entities
and not return_entities == (DatasetCollectionElement,)
):
q = q.join(HistoryDatasetAssociation).join(Dataset)
or (return_entities and return_entities != (DatasetCollectionElement,))
)
if needs_hda_join:
q = q.join(HistoryDatasetAssociation, HistoryDatasetAssociation.id == hda_join_col)
q = q.join(Dataset, Dataset.id == HistoryDatasetAssociation.dataset_id)
if dataset_permission_attributes:
q = q.join(DatasetPermissions)
q = q.join(DatasetPermissions, DatasetPermissions.dataset_id == Dataset.id)
q = (
q.add_columns(*attribute_columns(HistoryDatasetAssociation, hda_attributes))
.add_columns(*attribute_columns(Dataset, dataset_attributes))
@@ -7117,7 +7241,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable):
for entity in return_entities:
q = q.add_columns(entity)
if entity == DatasetCollectionElement:
q = q.filter(entity.id == dce.c.id)
q = q.filter(entity.id == dce_id_col)
q = q.order_by(*order_by_columns)
return q
@@ -7224,14 +7348,55 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable):
if ":" not in self.collection_type:
_populated_optimized = self.populated_state == DatasetCollection.populated_states.OK
else:
stmt = self._build_nested_collection_attributes_stmt(
collection_attributes=("populated_state",),
)
session = required_object_session(self)
for row in session.execute(stmt):
if any(state not in (DatasetCollection.populated_states.OK, None) for state in row):
_populated_optimized = False
break
is_postgres = session.bind and session.bind.dialect.name == "postgresql"
if is_postgres:
# Query intermediate collection IDs directly using the
# ARRAY walk pattern, then check their populated_state.
# Unlike the leaf-DCE ARRAY walk in
# _build_nested_collection_attributes_stmt, this
# correctly handles empty sub-collections (which have
# no leaf DCEs but may still have non-OK state, e.g.
# from skipped conditional workflow steps).
dce_table = DatasetCollectionElement.__table__
dc_table = DatasetCollection.__table__
n_intermediates = self.collection_type.count(":")
inner_dce = alias(dce_table)
child_ids_array = func.array(
select(inner_dce.c.child_collection_id)
.where(inner_dce.c.dataset_collection_id == self.id)
.scalar_subquery()
)
level_conditions = [dc_table.c.id == any_(child_ids_array)]
for _ in range(n_intermediates - 1):
next_dce = alias(dce_table)
child_ids_array = func.array(
select(next_dce.c.child_collection_id)
.where(next_dce.c.dataset_collection_id == any_(child_ids_array))
.scalar_subquery()
)
level_conditions.append(dc_table.c.id == any_(child_ids_array))
stmt = (
select(literal(1))
.select_from(dc_table)
.where(
or_(*level_conditions),
dc_table.c.populated_state != DatasetCollection.populated_states.OK,
)
.limit(1)
)
_populated_optimized = session.execute(stmt).first() is None
else:
stmt = self._build_nested_collection_attributes_stmt(
collection_attributes=("populated_state",),
)
for row in session.execute(stmt):
if any(state not in (DatasetCollection.populated_states.OK, None) for state in row):
_populated_optimized = False
break
self._populated_optimized = _populated_optimized
return self._populated_optimized
lib/galaxy/model/database_utils.py
+6 -8
View File
@@ -1,15 +1,12 @@
import sqlite3
from contextlib import contextmanager
from functools import lru_cache
from typing import (
NewType,
Optional,
)
from sqlalchemy import (
create_engine,
select,
update,
)
from sqlalchemy import create_engine
from sqlalchemy.engine import Engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.orm import object_session
@@ -20,7 +17,6 @@ from sqlalchemy.sql.expression import (
)
from galaxy.exceptions import ConfigurationError
from galaxy.model import Job
DbUrl = NewType("DbUrl", str)
@@ -147,19 +143,21 @@ def is_one_database(db1_url: str, db2_url: Optional[str]):
return not (db1_url and db2_url and db1_url != db2_url)
@lru_cache(maxsize=1)
def supports_returning(engine: Engine) -> bool:
"""
Return True if the database bound to `engine` supports the `RETURNING` SQL clause.
"""
stmt = update(Job).where(Job.id == -1).values(create_time=None).returning(Job.id)
stmt = text("UPDATE job SET tool_id=abc WHERE false RETURNING id")
return _statement_executed_without_error(stmt, engine)
@lru_cache(maxsize=1)
def supports_skip_locked(engine: Engine) -> bool:
"""
Return True if the database bound to `engine` supports the `SKIP_LOCKED` parameter.
"""
stmt = select(Job).where(Job.id == -1).with_for_update(skip_locked=True)
stmt = text("SELECT 1 FROM job WHERE false FOR UPDATE SKIP LOCKED")
return _statement_executed_without_error(stmt, engine)
lib/galaxy/tool_util/parameters/factory.py
+4 -2
View File
@@ -213,8 +213,10 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool
name=input_source.parse_name(),
**_common_param_kwargs(input_source),
)
elif param_type == "data":
optional = input_source.parse_optional()
elif param_type in ("data", "hidden_data"):
# hidden_data is broken without optional="true" (job runner rejects
# missing datasets); only known user is cufflinks which sets it.
optional = input_source.parse_optional() if param_type == "data" else True
multiple = input_source.get_bool("multiple", False)
return DataParameterModel(
type="data",
lib/galaxy/tool_util/verify/parse.py
+1 -1
View File
@@ -324,7 +324,7 @@ def _process_raw_inputs(
location = param_extra.get("location")
if param_type != "text":
param_value = split_if_str(param_value)
if param_type == "data":
if param_type in ("data", "hidden_data"):
if location and input_source.get_bool("multiple", False):
# We get the input/s from the location which can be a list of urls separated by commas
locations = split_if_str(location)
lib/galaxy/tools/parameters/basic.py
+12
View File
@@ -1533,6 +1533,10 @@ class ColumnListParameter(SelectToolParameter):
# otherwise read first row - assume is a header with tab separated names
if self.usecolnames:
dataset = other_values.get(self.data_ref, None)
if isinstance(dataset, HistoryDatasetCollectionAssociation):
dataset = dataset.to_hda_representative()
if isinstance(dataset, DatasetCollectionElement):
dataset = dataset.first_dataset_instance()
column_names = getattr(dataset, "metadata", None) and dataset.metadata.get_if_set("column_names")
if column_names:
try:
@@ -2710,6 +2714,14 @@ class HiddenDataToolParameter(HiddenToolParameter, DataToolParameter):
self.value = "None"
self.type = "hidden_data"
self.hidden = True
# hidden_data params are broken without optional="true" - the job runner's
# parameter validation rejects them when no dataset is provided. The only
# known tool using hidden_data (cufflinks) sets optional="true".
if not self.optional:
raise ParameterValueError(
'hidden_data parameters must declare optional="true" to function correctly',
self.name,
)
class BaseJsonToolParameter(ToolParameter):
lib/galaxy/tools/parameters/populate_model.py
+2 -6
View File
@@ -1,4 +1,3 @@
import logging
from typing import (
Any,
)
@@ -6,8 +5,6 @@ from typing import (
from galaxy.util.expressions import ExpressionContext
from .basic import ImplicitConversionRequired
log = logging.getLogger(__name__)
def populate_model(request_context, inputs, state_inputs, group_inputs: list[dict[str, Any]], other_values=None):
"""
@@ -50,6 +47,8 @@ def populate_model(request_context, inputs, state_inputs, group_inputs: list[dic
elif input.type == "section":
tool_dict = input.to_dict(request_context)
populate_model(request_context, input.inputs, group_state, tool_dict["inputs"], other_values)
elif input.type == "upload_dataset":
tool_dict = input.to_dict(request_context)
else:
try:
initial_value = input.get_initial_value(request_context, other_values)
@@ -63,9 +62,6 @@ def populate_model(request_context, inputs, state_inputs, group_inputs: list[dic
tool_dict = input.to_dict(request_context, other_values=other_values)
# This hack leads client to display a text field
tool_dict["textable"] = True
except Exception:
tool_dict = input.to_dict(request_context, other_values=other_values)
log.exception("tools::to_json() - Skipping parameter expansion '%s'", input.name)
if input_index >= len(group_inputs):
group_inputs.append(tool_dict)
else:
lib/galaxy/web/framework/middleware/error.py
+1 -1
View File
@@ -267,7 +267,7 @@ class CatchingIter:
if not self.start_checker.response_started:
self.start_checker("500 Internal Server Error", [("content-type", "text/html")], exc_info)
return response
return response.encode("utf-8", errors="ignore")
def close(self):
# This should at least print something to stderr if the
lib/galaxy/web/framework/middleware/profile.py
+2 -2
View File
@@ -67,7 +67,7 @@ class ProfileMiddleware:
prof = cProfile.Profile()
prof.runctx("run_app()", globals(), locals())
# Build up body with stats
body = "".join(body)
body = b"".join(body)
headers = catch_response[1]
content_type = response.header_value(headers, "content-type")
if not content_type.startswith("text/html"):
@@ -77,7 +77,7 @@ class ProfileMiddleware:
stats.strip_dirs()
stats.sort_stats("time", "calls")
output = pstats_as_html(stats, self.limit)
body += template % output
body += (template % output).encode("utf-8")
return [body]
lib/galaxy/web/framework/middleware/static.py
+1 -1
View File
@@ -54,7 +54,7 @@ class CacheableStaticURLParser(StaticURLParser):
headers: list[tuple[str, str]] = []
ETAG.update(headers, mytime)
start_response("304 Not Modified", headers)
return [""] # empty body
return [b""] # empty body
app = FileApp(full)
if self.cache_seconds:
app.cache_control(max_age=int(self.cache_seconds))
lib/galaxy_test/api/conftest.py
+17
View File
@@ -20,6 +20,11 @@ from galaxy_test.base.api_util import (
get_user_api_key,
)
from galaxy_test.base.env import setup_keep_outdir
from galaxy_test.base.mock_http_server import (
MockHTTPRequestHandler,
MockHttpServer,
start_mock_http_server,
)
from galaxy_test.base.populators import (
check_missing_tool,
DatasetCollectionPopulator,
@@ -177,3 +182,15 @@ def _requires_marker_to_effective_tool_id(anonymous_galaxy_interactor, marker):
all_tool_ids = get_tool_ids(anonymous_galaxy_interactor)
tool_id = [t for t in any_of_tool_ids if t in all_tool_ids][0]
return tool_id
@pytest.fixture(scope="session")
def mock_http_server():
if os.environ.get("GALAXY_TEST_EXTERNAL"):
yield MockHttpServer(base_url=None, handler_class=None, is_remote=True)
else:
server, base_url = start_mock_http_server()
try:
yield MockHttpServer(base_url=base_url, handler_class=MockHTTPRequestHandler, is_remote=False)
finally:
server.shutdown()
lib/galaxy_test/api/test_dataset_collections.py
+6 -4
View File
@@ -6,7 +6,6 @@ from urllib.parse import quote
from galaxy.tool_util_models.sample_sheet import SampleSheetColumnDefinitions
from galaxy.util import galaxy_root_path
from galaxy.util.unittest_utils import skip_if_github_down
from galaxy_test.base.api_asserts import (
assert_has_key,
assert_object_id_error,
@@ -917,16 +916,19 @@ class TestDatasetCollectionsApi(ApiTestCase):
object0 = element0["object"]
assert object0["state"] == "deferred"
@skip_if_github_down
def test_upload_collection_failed_expansion_url(self):
def test_upload_collection_failed_expansion_url(self, mock_http_server):
with self.dataset_populator.test_history(require_new=False) as history_id:
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed",
file_path="test-data/4.bed",
)
targets = [
{
"destination": {"type": "hdca"},
"elements_from": "bagit",
"collection_type": "list",
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed",
"url": url,
}
]
payload = {
lib/galaxy_test/api/test_datasets.py
+6 -4
View File
@@ -10,7 +10,6 @@ from galaxy.model.unittest_utils.store_fixtures import (
TEST_SOURCE_URI,
)
from galaxy.tool_util.verify.test_data import TestDataResolver
from galaxy.util.unittest_utils import skip_if_github_down
from galaxy_test.base.api_asserts import assert_has_keys
from galaxy_test.base.decorators import (
requires_admin,
@@ -879,11 +878,14 @@ class TestDatasetsApi(ApiTestCase):
assert len(sources) == 1
assert sources[0]["source_uri"] == TEST_SOURCE_URI
@skip_if_github_down
def test_display_application_link(self, history_id):
def test_display_application_link(self, history_id, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
file_path="test-data/1.bam",
)
item = {
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
"url": url,
"ext": "bam",
}
output = self.dataset_populator.fetch_hda(history_id, item)
lib/galaxy_test/api/test_libraries.py
+12 -7
View File
@@ -4,7 +4,6 @@ from galaxy.model.unittest_utils.store_fixtures import (
one_ld_library_model_store_dict,
TEST_LIBRARY_NAME,
)
from galaxy.util.unittest_utils import skip_if_github_down
from galaxy_test.base import api_asserts
from galaxy_test.base.decorators import requires_new_library
from galaxy_test.base.populators import (
@@ -317,16 +316,19 @@ class TestLibrariesApi(ApiTestCase):
self._assert_status_code_is(create_response, 400)
assert create_response.json()["err_msg"] == "Requested extension 'xxx' unknown, cannot upload dataset."
@skip_if_github_down
@requires_new_library
def test_fetch_failed_validation(self):
def test_fetch_failed_validation(self, mock_http_server):
# Exception handling is really rough here - we should be creating a dataset in error instead
# of just failing the job like this.
history_id, library, destination = self._setup_fetch_to_folder("single_url")
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed",
file_path="test-data/4.bed",
)
items = [
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed",
"url": url,
"MD5": "37b59762b59fff860460522d271bc112",
"name": "4.bed",
}
@@ -352,16 +354,19 @@ class TestLibrariesApi(ApiTestCase):
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["state"] == "error", dataset
@skip_if_github_down
@requires_new_library
def test_fetch_url_archive_to_folder(self):
def test_fetch_url_archive_to_folder(self, mock_http_server):
history_id, library, destination = self._setup_fetch_to_folder("single_url")
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip",
file_path="test-data/4.bed.zip",
)
targets = [
{
"destination": destination,
"items_from": "archive",
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed.zip",
"url": url,
}
]
payload = {
lib/galaxy_test/api/test_proxy.py
+26 -16
View File
@@ -7,11 +7,12 @@ from unittest.mock import (
import pytest
import requests
from galaxy.util.unittest_utils import skip_if_github_down
from galaxy_test.base.populators import DatasetPopulator
from ._framework import ApiTestCase
URL_TO_TEST = "https://github.com/galaxyproject/galaxy/raw/6e7427e8e2e070d4ab491a6cce5a59b6b6a62a08/test-data/4.bed.zip"
REMOTE_URL_TO_TEST = (
"https://github.com/galaxyproject/galaxy/raw/6e7427e8e2e070d4ab491a6cce5a59b6b6a62a08/test-data/4.bed.zip"
)
EXPECTED_HEADERS = {
"content-length": "198",
"content-type": "application/zip",
@@ -28,27 +29,36 @@ class TestProxyApi(ApiTestCase):
super().setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@skip_if_github_down
def test_proxy_head_request(self):
response = self._head(f"proxy?url={URL_TO_TEST}")
def _get_zip_url(self, mock_http_server):
return mock_http_server.get_url(
remote_url=REMOTE_URL_TO_TEST,
file_path="test-data/4.bed.zip",
content_type="application/zip",
support_head=True,
support_ranges=True,
)
def test_proxy_head_request(self, mock_http_server):
url = self._get_zip_url(mock_http_server)
response = self._head(f"proxy?url={url}")
self._assert_status_code_is_ok(response)
assert response.headers["content-length"] == EXPECTED_HEADERS["content-length"]
assert response.headers["content-type"] == EXPECTED_HEADERS["content-type"]
assert response.headers["accept-ranges"] == EXPECTED_HEADERS["accept-ranges"]
@skip_if_github_down
def test_proxy_get_request(self):
response = self._get(f"proxy?url={URL_TO_TEST}")
def test_proxy_get_request(self, mock_http_server):
url = self._get_zip_url(mock_http_server)
response = self._get(f"proxy?url={url}")
self._assert_status_code_is_ok(response)
assert response.headers["content-length"] == EXPECTED_HEADERS["content-length"]
assert response.headers["content-type"] == EXPECTED_HEADERS["content-type"]
assert response.headers["accept-ranges"] == EXPECTED_HEADERS["accept-ranges"]
assert len(response.content) == int(EXPECTED_HEADERS["content-length"])
@skip_if_github_down
def test_proxy_get_request_with_range(self):
def test_proxy_get_request_with_range(self, mock_http_server):
url = self._get_zip_url(mock_http_server)
request_range = "bytes=0-3"
response = self._get(f"proxy?url={URL_TO_TEST}", headers={"range": request_range})
response = self._get(f"proxy?url={url}", headers={"range": request_range})
self._assert_status_code_is(response, 206)
assert response.headers["accept-ranges"] == "bytes"
assert response.headers["content-length"] == "4"
@@ -56,7 +66,7 @@ class TestProxyApi(ApiTestCase):
assert response.content == ZIP_MAGIC_NUMBER
def test_anonymous_user_cannot_access_proxy(self):
response = self._get(f"proxy?url={URL_TO_TEST}", anon=True)
response = self._get(f"proxy?url={REMOTE_URL_TO_TEST}", anon=True)
self._assert_status_code_is(response, 403)
assert response.json()["err_msg"] == "Anonymous users are not allowed to access this endpoint"
@@ -83,12 +93,12 @@ class TestProxyApi(ApiTestCase):
@pytest.mark.parametrize(
"url_with_nonprintable",
[
f"\t{URL_TO_TEST}", # Tab at beginning
f"{URL_TO_TEST}\t", # Tab at end
f"\t{REMOTE_URL_TO_TEST}", # Tab at beginning
f"{REMOTE_URL_TO_TEST}\t", # Tab at end
"https://example.com/path\twith\ttab", # Tabs in path
"https://exam\tple.com/path", # Tab in domain
f"\r\n{URL_TO_TEST}", # CRLF at beginning
f"{URL_TO_TEST}\r\n", # CRLF at end
f"\r\n{REMOTE_URL_TO_TEST}", # CRLF at beginning
f"{REMOTE_URL_TO_TEST}\r\n", # CRLF at end
],
)
def test_url_with_nonprintable_characters(self, url_with_nonprintable: str):
lib/galaxy_test/api/test_tools.py
+6 -4
View File
@@ -20,7 +20,6 @@ from requests import (
from galaxy.tool_util.verify.interactor import ValidToolTestDict
from galaxy.util import galaxy_root_path
from galaxy.util.unittest_utils import skip_if_github_down
from galaxy_test.base import rules_test_data
from galaxy_test.base.api_asserts import (
assert_error_code_is,
@@ -365,13 +364,16 @@ class TestToolsApi(ApiTestCase, TestsTools):
assert "ex2" in option_values
@skip_without_tool("test_data_source")
@skip_if_github_down
def test_data_source_ok_request(self):
def test_data_source_ok_request(self, mock_http_server):
with self.dataset_populator.test_history() as history_id:
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed",
file_path="test-data/1.bed",
)
payload = self.dataset_populator.run_tool_payload(
tool_id="test_data_source",
inputs={
"URL": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed",
"URL": url,
"URL_method": "get",
"data_type": "bed",
},
lib/galaxy_test/api/test_tools_upload.py
+55 -29
View File
@@ -12,10 +12,6 @@ from galaxy.tool_util.verify.test_data import TestDataResolver
from galaxy.util import UNKNOWN
from galaxy.util.compression_utils import decompress_bytes_to_directory
from galaxy.util.hash_util import md5_hash_file
from galaxy.util.unittest_utils import (
skip_if_github_down,
skip_if_site_down,
)
from galaxy_test.base.constants import (
ONE_TO_SIX_ON_WINDOWS,
ONE_TO_SIX_WITH_SPACES,
@@ -341,18 +337,23 @@ class TestToolsUpload(ApiTestCase):
content = self.dataset_populator.get_history_dataset_content(history_id=history_id, dataset=dataset)
assert not content.startswith(">hg17")
@skip_if_github_down
def test_upload_multiple_mixed_success(self, history_id):
def test_upload_multiple_mixed_success(self, history_id, mock_http_server):
url_ok = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed",
file_path="test-data/1.bed",
)
url_error = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
status=404,
body="Not Found",
)
destination = {"type": "hdas"}
targets = [
{
"destination": destination,
"items": [
{"src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed"},
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
},
{"src": "url", "url": url_ok},
{"src": "url", "url": url_error},
],
}
]
@@ -371,18 +372,24 @@ class TestToolsUpload(ApiTestCase):
assert output0["state"] == "ok"
assert output1["state"] == "error"
@skip_if_github_down
def test_fetch_bam_file_from_url_with_extension_set(self, history_id):
def test_fetch_bam_file_from_url_with_extension_set(self, history_id, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
file_path="test-data/1.bam",
)
item = {
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bam",
"url": url,
"ext": "bam",
}
output = self.dataset_populator.fetch_hda(history_id, item)
self.dataset_populator.get_history_dataset_details(history_id, dataset=output, assert_ok=True)
@skip_if_github_down
def test_fetch_html_from_url(self, history_id):
def test_fetch_html_from_url(self, history_id, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/html_file.txt",
file_path="test-data/html_file.txt",
)
destination = {"type": "hdas"}
targets = [
{
@@ -390,7 +397,7 @@ class TestToolsUpload(ApiTestCase):
"items": [
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/html_file.txt",
"url": url,
},
],
}
@@ -409,13 +416,18 @@ class TestToolsUpload(ApiTestCase):
assert dataset["state"] == "error"
assert dataset["name"] == "html_file.txt"
@skip_if_site_down("https://httpstat.us")
def test_abort_fetch_job(self, history_id):
def test_abort_fetch_job(self, history_id, mock_http_server):
# This should probably be an integration test that also verifies
# that the celery chord is properly canceled.
url = mock_http_server.get_url(
remote_url="https://httpstat.us/200?sleep=10000",
status=200,
body="OK",
sleep_ms=10000,
)
item = {
"src": "url",
"url": "https://httpstat.us/200?sleep=10000",
"url": url,
"ext": "txt",
}
destination = {"type": "hdas"}
@@ -974,9 +986,13 @@ class TestToolsUpload(ApiTestCase):
with pytest.raises(AssertionError):
self._upload("https://foo.invalid", assert_ok=False)
@skip_if_site_down("https://usegalaxy.org")
def test_upload_from_404_url(self):
history_id, new_dataset = self._upload("https://usegalaxy.org/bla123", assert_ok=False)
def test_upload_from_404_url(self, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://usegalaxy.org/bla123",
status=404,
body="Not Found",
)
history_id, new_dataset = self._upload(url, assert_ok=False)
dataset_details = self.dataset_populator.get_history_dataset_details(
history_id, dataset_id=new_dataset["id"], assert_ok=False
)
@@ -984,14 +1000,24 @@ class TestToolsUpload(ApiTestCase):
dataset_details["state"] == "error"
), f"expected dataset state to be 'error', but got '{dataset_details['state']}'"
@skip_if_site_down("https://usegalaxy.org")
def test_upload_from_valid_url(self):
history_id, new_dataset = self._upload("https://usegalaxy.org/api/version")
def test_upload_from_valid_url(self, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://usegalaxy.org/api/version",
status=200,
body='{"version_major": "mock"}',
content_type="application/json",
)
history_id, new_dataset = self._upload(url)
self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=True)
@skip_if_site_down("https://usegalaxy.org")
def test_upload_from_valid_url_spaces(self):
history_id, new_dataset = self._upload(" https://usegalaxy.org/api/version ")
def test_upload_from_valid_url_spaces(self, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://usegalaxy.org/api/version",
status=200,
body='{"version_major": "mock"}',
content_type="application/json",
)
history_id, new_dataset = self._upload(f" {url} ")
self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=True)
def test_upload_and_validate_invalid(self):
lib/galaxy_test/api/test_workflows.py
+6 -3
View File
@@ -1496,9 +1496,12 @@ steps:
other_import_response = self.__import_workflow(workflow_id)
self._assert_status_code_is(other_import_response, 403)
@skip_if_github_down
def test_url_import(self):
url = "https://raw.githubusercontent.com/galaxyproject/galaxy/release_19.09/test/base/data/test_workflow_1.ga"
def test_url_import(self, mock_http_server):
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/release_19.09/test/base/data/test_workflow_1.ga",
file_path="lib/galaxy_test/base/data/test_workflow_1.ga",
content_type="application/json",
)
workflow_id = self._post("workflows", data={"archive_source": url}).json()["id"]
workflow = self._download_workflow(workflow_id)
assert "TestWorkflow1" in workflow["name"]
lib/galaxy_test/base/mock_http_server.py
+214
View File
@@ -0,0 +1,214 @@
"""A local mock HTTP server for replacing external test dependencies.
Replaces httpstat.us, usegalaxy.org, and raw.githubusercontent.com URLs
with a local server. Supports configurable status codes, file serving,
HEAD requests, range requests, and response delays.
When targeting a remote Galaxy server (GALAXY_TEST_EXTERNAL), falls back to
the real external URLs with automatic skip-if-down behavior.
"""
import os
import re
import threading
import time
from dataclasses import (
dataclass,
field,
)
from http.server import (
BaseHTTPRequestHandler,
HTTPServer,
)
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse
import pytest
from galaxy.util.unittest_utils import is_site_up
@dataclass
class Route:
status: int = 200
body: bytes = b""
headers: dict[str, str] = field(default_factory=dict)
sleep_ms: int = 0
method: str = "GET"
support_head: bool = False
support_ranges: bool = False
class MockHTTPRequestHandler(BaseHTTPRequestHandler):
routes: dict[str, Route] = {}
def _handle_request(self) -> None:
route = self.routes.get(self.path)
if route is None:
self.send_response(404)
self.end_headers()
self.wfile.write(b"Not Found")
return
is_head = self.command == "HEAD"
if is_head and not route.support_head:
self.send_response(405)
self.end_headers()
return
if not is_head and route.method != self.command:
self.send_response(405)
self.end_headers()
return
if route.sleep_ms > 0:
time.sleep(route.sleep_ms / 1000.0)
body = route.body
# Handle range requests
range_header = self.headers.get("Range")
if range_header and route.support_ranges:
match = re.match(r"bytes=(\d+)-(\d*)", range_header)
if match:
start = int(match.group(1))
end = int(match.group(2)) if match.group(2) else len(body) - 1
total = len(body)
body = body[start : end + 1]
self.send_response(206)
for key, value in route.headers.items():
self.send_header(key, value)
self.send_header("Content-Length", str(len(body)))
self.send_header("Content-Range", f"bytes {start}-{end}/{total}")
self.send_header("Accept-Ranges", "bytes")
self.end_headers()
if not is_head:
self.wfile.write(body)
return
self.send_response(route.status)
for key, value in route.headers.items():
self.send_header(key, value)
if route.support_ranges:
self.send_header("Accept-Ranges", "bytes")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
if not is_head:
self.wfile.write(body)
def do_GET(self) -> None:
self._handle_request()
def do_HEAD(self) -> None:
self._handle_request()
def do_POST(self) -> None:
self._handle_request()
def log_message(self, format, *args) -> None:
pass
class MockHttpServer:
"""A local mock HTTP server with a per-test URL factory.
When running locally, registers routes on the mock server and returns local URLs.
When targeting a remote Galaxy (GALAXY_TEST_EXTERNAL), returns the remote URL
and skips the test if the remote site is down.
"""
def __init__(
self,
base_url: Optional[str],
handler_class: Optional[type[MockHTTPRequestHandler]],
is_remote: bool,
):
self.base_url = base_url
self._handler_class = handler_class
self.is_remote = is_remote
self._counter = 0
def get_url(
self,
*,
remote_url: str,
status: int = 200,
body: str | bytes = "",
file_path: Optional[str] = None,
content_type: str = "application/octet-stream",
sleep_ms: int = 0,
response_headers: Optional[dict[str, str]] = None,
request_method: str = "GET",
support_head: bool = False,
support_ranges: bool = False,
) -> str:
"""Register a mock endpoint and return its URL, or return remote_url with skip-if-down.
Args:
remote_url: The real URL to use when targeting a remote Galaxy server.
status: HTTP status code to return.
body: Response body (string or bytes). Ignored if file_path is set.
file_path: Path to a file whose contents will be served as the response body.
content_type: Content-Type header value.
sleep_ms: Delay in milliseconds before responding.
response_headers: Additional response headers.
request_method: HTTP method to match (GET, POST, etc.).
support_head: Whether to also respond to HEAD requests.
support_ranges: Whether to support Range request headers (HTTP 206).
"""
if self.is_remote:
if not is_site_up(remote_url):
pytest.skip(f"Remote site {remote_url} is down")
return remote_url
assert self._handler_class is not None
self._counter += 1
# Include filename in path so tools can sniff file extensions from the URL
if file_path is not None:
filename = os.path.basename(file_path)
else:
filename = os.path.basename(urlparse(remote_url).path)
path = f"/mock/{self._counter}/{filename}" if filename else f"/mock/{self._counter}"
if file_path is not None:
file_path_obj = Path(file_path)
if not file_path_obj.is_absolute():
# Resolve relative paths from the Galaxy project root
galaxy_root = Path(__file__).resolve().parents[3]
file_path_obj = galaxy_root / file_path_obj
encoded_body = file_path_obj.read_bytes()
elif isinstance(body, str):
encoded_body = body.encode()
else:
encoded_body = body
headers = {"Content-Type": content_type}
if response_headers:
headers.update(response_headers)
self._handler_class.routes[path] = Route(
status=status,
body=encoded_body,
headers=headers,
sleep_ms=sleep_ms,
method=request_method,
support_head=support_head,
support_ranges=support_ranges,
)
return f"{self.base_url}{path}"
def start_mock_http_server(host: str = "127.0.0.1", port: int = 0) -> tuple[HTTPServer, str]:
"""Start a mock HTTP server in a daemon thread.
Args:
host: Address to bind to. Defaults to 127.0.0.1.
port: Port to bind to. 0 means OS-assigned.
Returns:
Tuple of (server, base_url).
"""
server = HTTPServer((host, port), MockHTTPRequestHandler)
actual_port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
return server, f"http://{host}:{actual_port}"
packages/app/setup.cfg
+1
View File
@@ -85,6 +85,7 @@ python_requires = >=3.10
[options.extras_require]
test =
galaxy-test-base
mock-ssh-server
pkce
pykwalify
packages/files/setup.cfg
+1
View File
@@ -42,6 +42,7 @@ python_requires = >=3.10
[options.extras_require]
test =
galaxy-test-base
pytest
gcsfs
s3fs>=2023.1.0
packages/packages_by_dep_dag.txt
+4 -2
View File
@@ -2,7 +2,6 @@ tool_util_models
util
config
files
job_metrics
navigation
objectstore
@@ -10,10 +9,13 @@ schema
tool_shed_schema
tool_util
test_base
files
data
selenium
tours
test_base
auth
job_execution
test-data/rocrate-test.zip
BIN
View File
Binary file not shown.
test/functional/tools/parameters/gx_hidden_data.xml
+28
View File
@@ -0,0 +1,28 @@
<tool id="gx_hidden_data" name="gx_hidden_data" version="1.0.0">
<command><![CDATA[
#if $parameter
cat '$parameter' >> '$output'
#else
echo 'NO_DATA' >> '$output'
#end if
]]></command>
<inputs>
<param name="parameter" type="hidden_data" ext="data" optional="true" />
</inputs>
<outputs>
<data name="output" format="data" />
</outputs>
<tests>
<test>
<output name="output">
<assert_contents>
<has_line line="NO_DATA" />
</assert_contents>
</output>
</test>
<test>
<param name="parameter" value="1.bed" />
<output name="output" file="1.bed" />
</test>
</tests>
</tool>
test/unit/app/managers/test_markdown_export.py
+2 -4
View File
@@ -280,7 +280,7 @@ invocation_time(invocation_id=1)
invocation = self._new_invocation()
self.app.workflow_manager.get_invocation.side_effect = [invocation, invocation]
result = self._to_basic(example)
expectedtime = invocation.create_time.strftime("%Y-%m-%d, %H:%M:%S")
expectedtime = invocation.create_time.strftime("%Y-%m-%d, %H:%M:%S UTC")
assert f"\n {expectedtime}" in result
def test_job_parameters(self):
@@ -419,9 +419,7 @@ invocation_time(invocation_id=1)
_, result, extra_data = self._ready_export(example)
assert "invocations" in extra_data
assert "create_time" in extra_data["invocations"]["be8be0fd2ce547f6"]
assert extra_data["invocations"]["be8be0fd2ce547f6"]["create_time"] == invocation.create_time.strftime(
"%Y-%m-%d, %H:%M:%S"
)
assert extra_data["invocations"]["be8be0fd2ce547f6"]["create_time"] == invocation.create_time.isoformat()
def test_export_replaces_embedded_history_dataset_type(self):
hda = self._new_hda()
test/unit/app/tools/conftest.py
+14
View File
@@ -7,6 +7,20 @@ from galaxy.model import tool_shed_install
from galaxy.model.tool_shed_install import mapping
from galaxy.tool_shed.cache import ToolShedRepositoryCache
from galaxy.tool_util.toolbox.base import ToolConfRepository
from galaxy_test.base.mock_http_server import (
MockHTTPRequestHandler,
MockHttpServer,
start_mock_http_server,
)
@pytest.fixture(scope="session")
def mock_http_server():
server, base_url = start_mock_http_server()
try:
yield MockHttpServer(base_url=base_url, handler_class=MockHTTPRequestHandler, is_remote=False)
finally:
server.shutdown()
@pytest.fixture
test/unit/app/tools/test_column_parameters.py
+15
View File
@@ -57,6 +57,11 @@ class TestDataColumnParameter(BaseParameterTestCase):
self.other_attributes = "value='c2'"
assert "2" == self.param.get_initial_value(self.trans, {"input_tsv": self.build_ready_hda()})
def test_get_options_hdca_with_usecolnames(self):
self.other_attributes = 'use_header_names="true"'
options = self.param.get_options(self.trans, {"input_tsv": self.build_ready_hdca()})
assert len(options) > 0
def setUp(self):
super().setUp()
self.test_history = model.History()
@@ -89,6 +94,16 @@ class TestDataColumnParameter(BaseParameterTestCase):
ready_hda.state = "ok"
return ready_hda
def build_ready_hdca(self):
hda = self.build_ready_hda()
session = self.app.model.context
c1 = model.DatasetCollection(collection_type="list")
model.DatasetCollectionElement(collection=c1, element=hda, element_identifier="sample1", element_index=0)
hdca = model.HistoryDatasetCollectionAssociation(collection=c1, name="test_collection")
session.add(hdca)
session.commit()
return hdca
@property
def param(self):
if not self._param:
test/unit/app/tools/test_data_fetch.py
+66 -31
View File
@@ -10,7 +10,6 @@ from typing import Optional
import pytest
from galaxy.tools.data_fetch import main
from galaxy.util.unittest_utils import skip_if_github_down
B64_FOR_1_2_3 = b64encode(b"1 2 3").decode("utf-8")
URI_FOR_1_2_3 = f"base64://{B64_FOR_1_2_3}"
@@ -63,9 +62,12 @@ def test_simple_path_get(hash_value: str, error_message: Optional[str]):
assert "error_message" not in hda_result
@skip_if_github_down
def test_simple_uri_get():
with _execute_context() as execute_context:
def test_simple_uri_get(mock_http_server):
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed",
file_path="test-data/1.bed",
)
with _execute_context(allow_localhost=True) as execute_context:
request = {
"targets": [
{
@@ -75,7 +77,7 @@ def test_simple_uri_get():
"elements": [
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/1.bed",
"url": url,
}
],
}
@@ -212,9 +214,13 @@ def test_incorrect_sha1():
)
@skip_if_github_down
def test_deferred_uri_get():
with _execute_context() as execute_context:
def test_deferred_uri_get(mock_http_server):
url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
status=404,
body="Not Found",
)
with _execute_context(allow_localhost=True) as execute_context:
request = {
"targets": [
{
@@ -224,7 +230,7 @@ def test_deferred_uri_get():
"elements": [
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
"url": url,
"deferred": True,
}
],
@@ -262,9 +268,13 @@ def test_simple_list_path_get():
assert destination["object_id"] == 76
@skip_if_github_down
def test_hdas_single_url_error():
with _execute_context() as execute_context:
def test_hdas_single_url_error(mock_http_server):
url_12_bed = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
status=404,
body="Not Found",
)
with _execute_context(allow_localhost=True) as execute_context:
job_directory = execute_context.job_directory
example_path = os.path.join(job_directory, "example_file")
with open(example_path, "w") as f:
@@ -279,7 +289,7 @@ def test_hdas_single_url_error():
{"src": "path", "path": example_path},
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
"url": url_12_bed,
},
],
}
@@ -293,14 +303,16 @@ def test_hdas_single_url_error():
assert "error_message" not in elements[0]
assert "error_message" in elements[1]
error = elements[1]["error_message"]
assert (
"Failed to fetch url https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed" in error
)
assert f"Failed to fetch url {url_12_bed}" in error
@skip_if_github_down
def test_hdca_collection_element_failed():
with _execute_context() as execute_context:
def test_hdca_collection_element_failed(mock_http_server):
url_12_bed = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
status=404,
body="Not Found",
)
with _execute_context(allow_localhost=True) as execute_context:
job_directory = execute_context.job_directory
example_path = os.path.join(job_directory, "example_file")
with open(example_path, "w") as f:
@@ -315,7 +327,7 @@ def test_hdca_collection_element_failed():
{"src": "path", "path": example_path},
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
"url": url_12_bed,
},
],
}
@@ -325,14 +337,16 @@ def test_hdca_collection_element_failed():
output = _unnamed_output(execute_context)
assert "error_message" in output
error = output["error_message"]
assert (
"Failed to fetch url https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed" in error
)
assert f"Failed to fetch url {url_12_bed}" in error
@skip_if_github_down
def test_hdca_allow_failed_collections():
with _execute_context() as execute_context:
def test_hdca_allow_failed_collections(mock_http_server):
url_12_bed = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
status=404,
body="Not Found",
)
with _execute_context(allow_localhost=True) as execute_context:
job_directory = execute_context.job_directory
example_path = os.path.join(job_directory, "example_file")
with open(example_path, "w") as f:
@@ -348,7 +362,7 @@ def test_hdca_allow_failed_collections():
{"src": "path", "path": example_path},
{
"src": "url",
"url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed",
"url": url_12_bed,
},
],
}
@@ -363,9 +377,7 @@ def test_hdca_allow_failed_collections():
assert "error_message" not in elements[0]
assert "error_message" in elements[1]
error = elements[1]["error_message"]
assert (
"Failed to fetch url https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/12.bed" in error
)
assert f"Failed to fetch url {url_12_bed}" in error
def test_hdca_failed_expansion():
@@ -397,9 +409,32 @@ def test_hdca_failed_expansion():
@contextmanager
def _execute_context():
def _execute_context(allow_localhost=False):
job_directory = mkdtemp()
try:
if allow_localhost:
file_sources_path = os.path.join(job_directory, "file_sources.json")
with open(file_sources_path, "w") as f:
json.dump(
{
"file_sources": [
{"type": "http", "id": "stock_http"},
{"type": "base64", "id": "stock_base64"},
],
"config": {
"symlink_allowlist": [],
"fetch_url_allowlist": ["127.0.0.0/24"],
"library_import_dir": None,
"user_library_import_dir": None,
"ftp_upload_dir": None,
"ftp_upload_purge": True,
"tmp_dir": None,
"webdav_use_temp_files": None,
"listings_expiry_time": None,
},
},
f,
)
# temporarily set tempdir to non-existing location
# to make sure all intermediate files are created in the working
# directory
test/unit/data/test_galaxy_mapping.py
+55
View File
@@ -218,6 +218,61 @@ class TestMappings(BaseModelTestCase):
]
assert c4.dataset_elements == [dce1, dce2]
def test_nested_collection_attributes_duplicate_child_collection_id(self):
"""Regression test for CardinalityViolation when multiple parent elements
reference the same child collection (duplicate child_collection_id).
On PostgreSQL the ARRAY walk optimisation uses correlated scalar
subqueries to navigate from leaf elements back to their ancestors via
child_collection_id. When two parent DatasetCollectionElements share
the same child_collection_id those subqueries return more than one row,
causing 'more than one row returned by a subquery used as an expression'.
"""
u = model.User(email=random_email(), password="password")
h1 = model.History(name="History 1", user=u)
# Build a paired collection with two datasets.
pair = model.DatasetCollection(collection_type="paired")
forward = model.HistoryDatasetAssociation(
extension="txt", history=h1, create_dataset=True, sa_session=self.model.session
)
reverse = model.HistoryDatasetAssociation(
extension="bam", history=h1, create_dataset=True, sa_session=self.model.session
)
model.DatasetCollectionElement(collection=pair, element=forward, element_identifier="forward", element_index=0)
model.DatasetCollectionElement(collection=pair, element=reverse, element_identifier="reverse", element_index=1)
# Create a list:paired collection where the *same* paired collection
# is referenced by two parent elements (duplicate child_collection_id).
list_pair = model.DatasetCollection(collection_type="list:paired")
model.DatasetCollectionElement(
collection=list_pair, element=pair, element_identifier="sample1", element_index=0
)
model.DatasetCollectionElement(
collection=list_pair, element=pair, element_identifier="sample2", element_index=1
)
self.persist(u, h1, forward, reverse, pair, list_pair, commit=True, expunge=False)
self.model.session.flush()
# All query paths that use _build_nested_collection_attributes_stmt
# must tolerate the duplicate without raising CardinalityViolation.
stmt = list_pair._build_nested_collection_attributes_stmt(
element_attributes=("element_identifier",),
hda_attributes=("extension",),
dataset_attributes=("state",),
)
result = self.model.session.execute(stmt).all()
# The two leaf datasets should appear (potentially duplicated because
# the same child collection is reachable via two parents).
extensions_in_result = {r.extension for r in result}
assert extensions_in_result == {"txt", "bam"}
# dataset_states_and_extensions_summary must not raise either.
summary = list_pair.dataset_states_and_extensions_summary
assert "txt" in summary.extensions
assert "bam" in summary.extensions
def test_history_audit(self):
u = model.User(email=random_email(), password="password")
h1 = model.History(name="HistoryAuditHistory", user=u)
test/unit/files/conftest.py
+16
View File
@@ -0,0 +1,16 @@
import pytest
from galaxy_test.base.mock_http_server import (
MockHTTPRequestHandler,
MockHttpServer,
start_mock_http_server,
)
@pytest.fixture(scope="session")
def mock_http_server():
server, base_url = start_mock_http_server()
try:
yield MockHttpServer(base_url=base_url, handler_class=MockHTTPRequestHandler, is_remote=False)
finally:
server.shutdown()
test/unit/files/test_remotezip.py
+16 -6
View File
@@ -1,9 +1,10 @@
import ipaddress
import os
import pytest
from galaxy.exceptions import AdminRequiredException
from galaxy.util.unittest_utils import skip_if_github_down
from galaxy.files.models import FileSourcePluginsConfig
from ._util import (
assert_realizes_contains,
configured_file_sources,
@@ -13,19 +14,28 @@ from ._util import (
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "remotezip_file_sources_conf.yml")
LOCALHOST_ALLOWLIST = FileSourcePluginsConfig(
fetch_url_allowlist=[ipaddress.ip_network("127.0.0.0/24")],
)
@skip_if_github_down
def test_file_source():
remote_zip_url = "https://raw.githubusercontent.com/davelopez/ro-crate-zip-explorer/refs/heads/main/tests/test-data/rocrate-test.zip"
def test_file_source(mock_http_server):
zip_url = mock_http_server.get_url(
remote_url="https://raw.githubusercontent.com/davelopez/ro-crate-zip-explorer/refs/heads/main/tests/test-data/rocrate-test.zip",
file_path="test-data/rocrate-test.zip",
content_type="application/zip",
support_ranges=True,
support_head=True,
)
header_offset = 15875
compression_method = 8
compress_size = 582
expected_contents = "class: GalaxyWorkflow"
test_url = f"zip://extract?source={remote_zip_url}&header_offset={header_offset}&compress_size={compress_size}&compression_method={compression_method}"
test_url = f"zip://extract?source={zip_url}&header_offset={header_offset}&compress_size={compress_size}&compression_method={compression_method}"
user_context = user_context_fixture()
file_sources = configured_file_sources(FILE_SOURCES_CONF)
file_sources_config = LOCALHOST_ALLOWLIST if not mock_http_server.is_remote else None
file_sources = configured_file_sources(FILE_SOURCES_CONF, file_sources_config=file_sources_config)
file_source_pair = file_sources.get_file_source_path(test_url)
assert file_source_pair.path == test_url
test/unit/tool_util/parameter_specification.yml
+26 -12
View File
@@ -1368,11 +1368,11 @@ gx_data:
gx_data_optional:
request_valid:
request_valid: &data_optional_request_valid
- parameter: {src: hda, id: abcdabcd}
- parameter: null
- {}
request_invalid:
request_invalid: &data_optional_request_invalid
- parameter: {src: hda, id: 5}
- parameter: {src: hdca, id: abcdabcd}
- parameter: {src: fooda, id: abcdabcd}
@@ -1381,11 +1381,11 @@ gx_data_optional:
- parameter: true
- parameter: 5
- parameter: "5"
request_internal_valid:
request_internal_valid: &data_optional_request_internal_valid
- parameter: {src: hda, id: 5}
- parameter: null
- {}
request_internal_invalid:
request_internal_invalid: &data_optional_request_internal_invalid
- parameter: {src: hda, id: abcdabcd}
- parameter: {src: hdca, id: abcdabcd}
- parameter: {src: fooda, id: abcdabcd}
@@ -1394,10 +1394,10 @@ gx_data_optional:
- parameter: true
- parameter: 5
- parameter: "5"
job_internal_valid:
job_internal_valid: &data_optional_job_internal_valid
- parameter: null
- parameter: {src: hda, id: 5}
job_internal_invalid:
job_internal_invalid: &data_optional_job_internal_invalid
- {}
job_runtime_valid:
- parameter: {class: File, basename: "f.txt", location: "step_input://0",
@@ -1407,27 +1407,41 @@ gx_data_optional:
- parameter: {src: hda, id: 7}
- parameter: {class: File}
- {}
workflow_step_valid:
workflow_step_valid: &data_optional_workflow_step_valid
- {}
workflow_step_invalid:
workflow_step_invalid: &data_optional_workflow_step_invalid
- {src: hda, id: abcdabcd}
- {src: hda, id: 7}
- parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]}
- parameter: {__class__: 'ConnectedValue'}
workflow_step_linked_valid:
workflow_step_linked_valid: &data_optional_workflow_step_linked_valid
- {}
- parameter: {__class__: 'ConnectedValue'}
workflow_step_linked_invalid:
workflow_step_linked_invalid: &data_optional_workflow_step_linked_invalid
- {src: hda, id: abcdabcd}
- {src: hda, id: 7}
- parameter: {__class__: "Batch", values: [{src: hdca, id: 5}]}
- parameter: {__class__: 'ConnectedValueX'}
test_case_xml_valid:
test_case_xml_valid: &data_optional_test_case_xml_valid
- {}
- parameter: {class: "File", path: "1.bed"}
test_case_xml_invalid:
test_case_xml_invalid: &data_optional_test_case_xml_invalid
- parameter: {class: "NotAFile", path: "1.bed"}
gx_hidden_data:
request_valid: *data_optional_request_valid
request_invalid: *data_optional_request_invalid
request_internal_valid: *data_optional_request_internal_valid
request_internal_invalid: *data_optional_request_internal_invalid
job_internal_valid: *data_optional_job_internal_valid
job_internal_invalid: *data_optional_job_internal_invalid
workflow_step_valid: *data_optional_workflow_step_valid
workflow_step_invalid: *data_optional_workflow_step_invalid
workflow_step_linked_valid: *data_optional_workflow_step_linked_valid
workflow_step_linked_invalid: *data_optional_workflow_step_linked_invalid
test_case_xml_valid: *data_optional_test_case_xml_valid
test_case_xml_invalid: *data_optional_test_case_xml_invalid
gx_data_multiple:
request_valid:
- parameter: {src: hda, id: abcdabcd}