Merge branch 'release_24.0' into dev

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@galaxyproject/galaxy-client",
-  "version": "23.1.1",
+  "version": "24.0.0",
   "description": "Galaxy client application build system",
   "keywords": [
     "galaxy"

client/src/components/Form/Elements/FormData/FormData.vue

@@ -14,6 +14,7 @@ import { orList } from "@/utils/strings";
 import type { DataOption } from "./types";
 import { BATCH, SOURCE, VARIANTS } from "./variants";
 
+import FormSelection from "../FormSelection.vue";
 import FormSelect from "@/components/Form/Elements/FormSelect.vue";
 
 library.add(faCopy, faFile, faFolder, faCaretDown, faCaretUp, faExclamation, faLink, faUnlink);
@@ -502,7 +503,7 @@ const noOptionsWarningMessage = computed(() => {
         </div>
 
         <FormSelect
-            v-if="currentVariant"
+            v-if="currentVariant && !currentVariant.multiple"
             v-model="currentValue"
             class="align-self-start"
             :multiple="currentVariant.multiple"
@@ -515,6 +516,12 @@ const noOptionsWarningMessage = computed(() => {
                 </BAlert>
             </template>
         </FormSelect>
+        <FormSelection
+            v-else-if="currentVariant?.multiple"
+            v-model="currentValue"
+            :data="formattedOptions"
+            optional
+            multiple />
 
         <template v-if="currentVariant && currentVariant.batch !== BATCH.DISABLED">
             <BFormCheckbox

client/src/components/Form/Elements/FormSelectMany/FormSelectMany.vue

@@ -65,7 +65,13 @@ const searchRegex = computed(() => {
 /** Wraps value prop so it can be set, and always returns an array */
 const selected = computed({
     get() {
-        return Array.isArray(props.value) ? props.value : [props.value];
+        if (props.value === null) {
+            return [];
+        } else if (Array.isArray(props.value)) {
+            return props.value;
+        } else {
+            return [props.value];
+        }
     },
     set(value) {
         emit("input", value);
@@ -142,7 +148,15 @@ async function deselectOption(event: MouseEvent, index: number) {
         const [option] = selectedOptionsFiltered.value.splice(index, 1);
 
         if (option) {
-            const i = selected.value.indexOf(option.value);
+            const i = selected.value.findIndex((selectedValue) => {
+                if (typeof selectedValue === "string") {
+                    return selectedValue === option.value;
+                } else if (typeof selectedValue === "object" && typeof option.value === "object") {
+                    // in case values are objects, compare their ids (if they have the 'id' property)
+                    return selectedValue?.id === option.value?.id;
+                }
+                return false;
+            });
             selected.value = selected.value.flatMap((value, index) => (index === i ? [] : [value]));
         }
 

client/src/components/Form/Elements/FormSelection.vue

@@ -82,7 +82,10 @@ watch(
         }
 
         if (newValue === "none") {
-            if ((Array.isArray(props.value) && props.value.length >= 15) || props.options.length >= 500) {
+            if (
+                (Array.isArray(props.value) && props.value.length >= 15) ||
+                (props.options && props.options.length >= 500)
+            ) {
                 useMany.value = true;
             } else {
                 useMany.value = false;

client/src/components/History/HistoryView.vue

@@ -8,7 +8,6 @@
             <b-button
                 v-if="userOwnsHistory"
                 size="sm"
-                variant="outline-info"
                 :title="setAsCurrentTitle"
                 :disabled="isSetAsCurrentDisabled"
                 data-description="switch to history button"
@@ -20,13 +19,14 @@
                 v-if="canImportHistory"
                 v-b-modal:copy-history-modal
                 size="sm"
-                variant="outline-info"
                 title="Import this history"
                 data-description="import history button">
                 Import this history
             </b-button>
         </div>
 
+        <b-alert :show="copySuccess"> History imported and set to your active history. </b-alert>
+
         <CollectionPanel
             v-if="selectedCollections.length && selectedCollections[0].history_id == id"
             :history="history"
@@ -40,7 +40,7 @@
             filterable
             @view-collection="onViewCollection" />
 
-        <CopyModal id="copy-history-modal" :history="history" />
+        <CopyModal id="copy-history-modal" :history="history" @ok="copyOkay" />
     </div>
 </template>
 
@@ -69,6 +69,7 @@ export default {
     data() {
         return {
             selectedCollections: [],
+            copySuccess: false,
         };
     },
     computed: {
@@ -129,6 +130,9 @@ export default {
         onViewCollection(collection) {
             this.selectedCollections = [...this.selectedCollections, collection];
         },
+        copyOkay() {
+            this.copySuccess = true;
+        },
     },
 };
 </script>

client/src/components/Markdown/MarkdownToolBox.vue

@@ -252,6 +252,35 @@ export default {
         this.getVisualizations();
     },
     methods: {
+        getSteps() {
+            const steps = [];
+            this.steps &&
+                Object.values(this.steps).forEach((step) => {
+                    if (step.label) {
+                        steps.push(step.label);
+                    }
+                });
+            return steps;
+        },
+        getOutputs(filterByType = undefined) {
+            const outputLabels = [];
+            this.steps &&
+                Object.values(this.steps).forEach((step) => {
+                    step.workflow_outputs.forEach((workflowOutput) => {
+                        if (workflowOutput.label) {
+                            if (!filterByType || this.stepOutputMatchesType(step, workflowOutput, filterByType)) {
+                                outputLabels.push(workflowOutput.label);
+                            }
+                        }
+                    });
+                });
+            return outputLabels;
+        },
+        stepOutputMatchesType(step, workflowOutput, type) {
+            return Boolean(
+                step.outputs.find((output) => output.name === workflowOutput.output_name && output.type === type)
+            );
+        },
         getArgumentTitle(argumentName) {
             return (
                 argumentName[0].toUpperCase() +
@@ -311,11 +340,13 @@ export default {
         onHistoryDatasetId(argumentName) {
             this.selectedArgumentName = argumentName;
             this.selectedType = "history_dataset_id";
+            this.selectedLabels = this.getOutputs("data");
             this.selectedShow = true;
         },
         onHistoryCollectionId(argumentName) {
             this.selectedArgumentName = argumentName;
             this.selectedType = "history_dataset_collection_id";
+            this.selectedLabels = this.getOutputs("collection");
             this.selectedShow = true;
         },
         onWorkflowId(argumentName) {

client/src/components/plugins/localization.js

@@ -12,11 +12,14 @@ function localizeDirective(l) {
         // TODO consider using a different hook if we need dynamic updates in content translation
         bind(el, binding, vnode) {
             el.childNodes.forEach((node) => {
+                // trim for lookup, but put back whitespace after
+                const leadingSpace = node.textContent.match(/^\s*/)[0];
+                const trailingSpace = node.textContent.match(/\s*$/)[0];
                 const standardizedContent = node.textContent
                     .replace(newlineMatch, " ")
                     .replace(doublespaces, " ")
                     .trim();
-                node.textContent = l(standardizedContent);
+                node.textContent = leadingSpace + l(standardizedContent) + trailingSpace;
             });
         },
     };

lib/galaxy/authnz/managers.py

@@ -347,9 +347,8 @@ class AuthnzManager:
             if refreshed:
                 log.debug(f"Refreshed user token via `{auth.provider}` identity provider")
             return True
-        except Exception as e:
-            msg = f"An error occurred when refreshing user token: {e}"
-            log.error(msg)
+        except Exception:
+            log.exception("An error occurred when refreshing user token")
             return False
 
     def refresh_expiring_oidc_tokens(self, trans, user=None):

lib/galaxy/config/sample/file_sources_conf.yml.sample

@@ -202,10 +202,14 @@
   doc: Make sure to define this generic drs file source if you have defined any other drs file sources, or stock drs download capability will be disabled.
 
 - type: inveniordm
-  id: invenio
-  doc: Invenio RDM turn-key research data management repository
-  label: Invenio RDM Demo Repository
+  id: invenio_sandbox
+  doc: This is the Sandbox instance of Invenio. It is used for testing purposes only, content is NOT preserved. DOIs created in this instance are not real and will not resolve.
+  label: Invenio RDM Sandbox Repository (TESTING ONLY)
   url: https://inveniordm.web.cern.ch/
+  token: ${user.user_vault.read_secret('preferences/invenio_sandbox/token')}
+  # token: ${user.preferences['invenio_sandbox|token']} # Alternatively use this for retrieving the token from user preferences instead of the Vault
+  public_name: ${user.preferences['invenio_sandbox|public_name']}
+  writable: true
 
 - type: onedata
   id: onedata1

lib/galaxy/files/sources/_rdm.py

@@ -1,6 +1,5 @@
 import logging
 from typing import (
-    cast,
     List,
     NamedTuple,
     Optional,
@@ -26,6 +25,7 @@ OptionalUserContext = Optional[ProvidesUserFileSourcesUserContext]
 class RDMFilesSourceProperties(FilesSourceProperties):
     url: str
     token: str
+    public_name: str
 
 
 class RecordFilename(NamedTuple):
@@ -80,7 +80,9 @@ class RDMRepositoryInteractor:
         """
         raise NotImplementedError()
 
-    def create_draft_record(self, title: str, user_context: OptionalUserContext = None):
+    def create_draft_record(
+        self, title: str, public_name: Optional[str] = None, user_context: OptionalUserContext = None
+    ):
         """Creates a draft record (directory) in the repository with basic metadata.
 
         The metadata is usually just the title of the record and the user that created it.
@@ -138,11 +140,10 @@ class RDMFilesSource(BaseFilesSource):
 
     def __init__(self, **kwd: Unpack[FilesSourceProperties]):
         props = self._parse_common_config_opts(kwd)
-        base_url = props.get("url", None)
+        base_url = props.get("url")
         if not base_url:
             raise Exception("URL for RDM repository must be provided in configuration")
         self._repository_url = base_url
-        self._token = props.get("token", None)
         self._props = props
         self._repository_interactor = self.get_repository_interactor(base_url)
 
@@ -150,10 +151,6 @@ class RDMFilesSource(BaseFilesSource):
     def repository(self) -> RDMRepositoryInteractor:
         return self._repository_interactor
 
-    @property
-    def token(self) -> Optional[str]:
-        return self._token if self._token and not self._token.startswith("$") else None
-
     def get_repository_interactor(self, repository_url: str) -> RDMRepositoryInteractor:
         """Returns an interactor compatible with the given repository URL.
 
@@ -190,25 +187,23 @@ class RDMFilesSource(BaseFilesSource):
     def get_record_id_from_path(self, source_path: str) -> str:
         return self.parse_path(source_path, record_id_only=True).record_id
 
-    def _serialization_props(self, user_context: OptionalUserContext = None) -> RDMFilesSourceProperties:
+    def _serialization_props(self, user_context: OptionalUserContext = None):
         effective_props = {}
         for key, val in self._props.items():
             effective_props[key] = self._evaluate_prop(val, user_context=user_context)
-        effective_props["url"] = self._repository_url
-        effective_props["token"] = self.safe_get_authorization_token(user_context)
-        return cast(RDMFilesSourceProperties, effective_props)
+        return effective_props
 
     def get_authorization_token(self, user_context: OptionalUserContext) -> str:
-        token = self.token
-        if not token and user_context:
-            vault = user_context.user_vault if user_context else None
-            token = vault.read_secret(f"preferences/{self.id}/token") if vault else None
-        if token is None:
-            raise AuthenticationRequired(f"No authorization token provided in user's settings for '{self.label}'")
+        token = None
+        if user_context:
+            effective_props = self._serialization_props(user_context)
+            token = effective_props.get("token")
+        if not token:
+            raise AuthenticationRequired(
+                f"Please provide a personal access token in your user's preferences for '{self.label}'"
+            )
         return token
 
-    def safe_get_authorization_token(self, user_context: OptionalUserContext) -> Optional[str]:
-        try:
-            return self.get_authorization_token(user_context)
-        except AuthenticationRequired:
-            return None
+    def get_public_name(self, user_context: OptionalUserContext) -> Optional[str]:
+        effective_props = self._serialization_props(user_context)
+        return effective_props.get("public_name")

lib/galaxy/files/sources/invenio.py

@@ -135,7 +135,8 @@ class InvenioRDMFilesSource(RDMFilesSource):
         user_context: OptionalUserContext = None,
         opts: Optional[FilesSourceOptions] = None,
     ) -> Entry:
-        record = self.repository.create_draft_record(entry_data["name"], user_context=user_context)
+        public_name = self.get_public_name(user_context)
+        record = self.repository.create_draft_record(entry_data["name"], public_name, user_context=user_context)
         return {
             "uri": self.repository.to_plugin_uri(record["id"]),
             "name": record["metadata"]["title"],
@@ -198,9 +199,11 @@ class InvenioRepositoryInteractor(RDMRepositoryInteractor):
         response_data = self._get_response(user_context, request_url)
         return self._get_record_files_from_response(record_id, response_data)
 
-    def create_draft_record(self, title: str, user_context: OptionalUserContext = None) -> RemoteDirectory:
+    def create_draft_record(
+        self, title: str, public_name: Optional[str] = None, user_context: OptionalUserContext = None
+    ) -> RemoteDirectory:
         today = datetime.date.today().isoformat()
-        creator = self._get_creator_from_user_context(user_context)
+        creator = self._get_creator_from_public_name(public_name)
         create_record_request = {
             "files": {"enabled": True},
             "metadata": {
@@ -360,10 +363,9 @@ class InvenioRepositoryInteractor(RDMRepositoryInteractor):
                 )
         return rval
 
-    def _get_creator_from_user_context(self, user_context: OptionalUserContext):
-        public_name = self.get_user_preference_by_key("public_name", user_context)
-        family_name = "Galaxy User"
+    def _get_creator_from_public_name(self, public_name: Optional[str] = None) -> Creator:
         given_name = "Anonymous"
+        family_name = "Galaxy User"
         if public_name:
             tokens = public_name.split(", ")
             if len(tokens) == 2:
@@ -371,12 +373,16 @@ class InvenioRepositoryInteractor(RDMRepositoryInteractor):
                 given_name = tokens[1]
             else:
                 given_name = public_name
-        return {"person_or_org": {"family_name": family_name, "given_name": given_name, "type": "personal"}}
-
-    def get_user_preference_by_key(self, key: str, user_context: OptionalUserContext):
-        preferences = user_context.preferences if user_context else None
-        value = preferences.get(f"{self.plugin.id}|{key}", None) if preferences else None
-        return value
+        return {
+            "person_or_org": {
+                "name": f"{given_name} {family_name}",
+                "family_name": family_name,
+                "given_name": given_name,
+                "type": "personal",
+                "identifiers": [],
+            },
+            "affiliations": [],
+        }
 
     def _get_response(
         self, user_context: OptionalUserContext, request_url: str, params: Optional[Dict[str, Any]] = None

lib/galaxy/managers/hdas.py

@@ -68,6 +68,7 @@ from galaxy.structured_app import (
     MinimalManagerApp,
     StructuredApp,
 )
+from galaxy.util.compression_utils import get_fileobj
 
 log = logging.getLogger(__name__)
 
@@ -310,11 +311,13 @@ class HDAManager(
         # For now, cannot get data from non-text datasets.
         if not isinstance(hda.datatype, datatypes.data.Text):
             return truncated, hda_data
-        if not os.path.exists(hda.get_file_name()):
+        file_path = hda.get_file_name()
+        if not os.path.exists(file_path):
             return truncated, hda_data
 
-        truncated = preview and os.stat(hda.get_file_name()).st_size > MAX_PEEK_SIZE
-        hda_data = open(hda.get_file_name()).read(MAX_PEEK_SIZE)
+        truncated = preview and os.stat(file_path).st_size > MAX_PEEK_SIZE
+        with get_fileobj(file_path) as fh:
+            hda_data = fh.read(MAX_PEEK_SIZE)
         return truncated, hda_data
 
     # .... annotatable

lib/galaxy/managers/workflows.py

@@ -727,7 +727,8 @@ class WorkflowContentsManager(UsesAnnotations):
             trans.tag_handler.set_tags_from_list(
                 trans.user,
                 stored_workflow,
-                data.get("tags", []),
+                data["tags"],
+                flush=False,
             )
 
         if workflow_update_options.update_stored_workflow_attributes:

lib/galaxy/model/__init__.py

@@ -8096,10 +8096,20 @@ class WorkflowStep(Base, RepresentById):
             copied_step.annotations = annotations
 
         if subworkflow := self.subworkflow:
-            copied_subworkflow = subworkflow.copy()
-            copied_step.subworkflow = copied_subworkflow
-            for subworkflow_step, copied_subworkflow_step in zip(subworkflow.steps, copied_subworkflow.steps):
-                subworkflow_step_mapping[subworkflow_step.id] = copied_subworkflow_step
+            stored_subworkflow = subworkflow.stored_workflow
+            if stored_subworkflow and stored_subworkflow.user == user:
+                # This should be fine and reduces the number of stored subworkflows
+                copied_step.subworkflow = subworkflow
+            else:
+                # Can this even happen, building a workflow with a subworkflow you don't own ?
+                copied_subworkflow = subworkflow.copy()
+                stored_workflow = StoredWorkflow(
+                    user, name=copied_subworkflow.name, workflow=copied_subworkflow, hidden=True
+                )
+                copied_subworkflow.stored_workflow = stored_workflow
+                copied_step.subworkflow = copied_subworkflow
+                for subworkflow_step, copied_subworkflow_step in zip(subworkflow.steps, copied_subworkflow.steps):
+                    subworkflow_step_mapping[subworkflow_step.id] = copied_subworkflow_step
 
         for old_conn, new_conn in zip(self.input_connections, copied_step.input_connections):
             new_conn.input_step_input = copied_step.get_or_add_input(old_conn.input_name)
@@ -8661,11 +8671,11 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, Serializabl
         # That probably isn't good.
         workflow_output = self.workflow.workflow_output_for(label)
         if workflow_output:
-            raise Exception(
+            raise galaxy.exceptions.MessageException(
                 f"Failed to find workflow output named [{label}], one was defined but none registered during execution."
             )
         else:
-            raise Exception(
+            raise galaxy.exceptions.MessageException(
                 f"Failed to find workflow output named [{label}], workflow doesn't define output by that name - valid names are {self.workflow.workflow_output_labels}."
             )
 

lib/galaxy/tools/actions/__init__.py

@@ -17,8 +17,12 @@ from typing import (
 from packaging.version import Version
 
 from galaxy import model
-from galaxy.exceptions import ItemAccessibilityException
+from galaxy.exceptions import (
+    ItemAccessibilityException,
+    RequestParameterInvalidException,
+)
 from galaxy.job_execution.actions.post import ActionBox
+from galaxy.managers.context import ProvidesHistoryContext
 from galaxy.model import (
     HistoryDatasetAssociation,
     Job,
@@ -98,7 +102,7 @@ class DefaultToolAction(ToolAction):
         self,
         tool,
         param_values,
-        trans,
+        trans: ProvidesHistoryContext,
         history,
         current_user_roles=None,
         dataset_collection_elements=None,
@@ -256,6 +260,10 @@ class DefaultToolAction(ToolAction):
                 for ext in extensions:
                     if ext:
                         datatype = trans.app.datatypes_registry.get_datatype_by_extension(ext)
+                        if not datatype:
+                            raise RequestParameterInvalidException(
+                                f"Extension '{ext}' unknown, cannot use dataset collection as input"
+                            )
                         if not datatype.matches_any(input.formats):
                             conversion_required = True
                             break

lib/galaxy/tools/parameters/basic.py

@@ -897,6 +897,11 @@ class BaseURLToolParameter(HiddenToolParameter):
         return d
 
 
+def iter_to_string(iterable: typing.Iterable[typing.Any]) -> typing.Generator[str, None, None]:
+    for item in iterable:
+        yield str(item)
+
+
 class SelectToolParameter(ToolParameter):
     """
     Parameter that takes on one (or many) or a specific set of values.
@@ -1046,8 +1051,9 @@ class SelectToolParameter(ToolParameter):
             elif set(value).issubset(set(fallback_values.keys())):
                 return [fallback_values[v] for v in value]
             else:
+                invalid_options = iter_to_string(set(value) - set(legal_values))
                 raise ParameterValueError(
-                    f"invalid options ({','.join(set(value) - set(legal_values))!r}) were selected (valid options: {','.join(legal_values)})",
+                    f"invalid options ({','.join(invalid_options)!r}) were selected (valid options: {','.join(iter_to_string(legal_values))})",
                     self.name,
                     is_dynamic=self.is_dynamic,
                 )
@@ -1071,7 +1077,7 @@ class SelectToolParameter(ToolParameter):
                 return value
             else:
                 raise ParameterValueError(
-                    f"an invalid option ({value!r}) was selected (valid options: {','.join(legal_values)})",
+                    f"an invalid option ({value!r}) was selected (valid options: {','.join(iter_to_string(legal_values))})",
                     self.name,
                     value,
                     is_dynamic=self.is_dynamic,

lib/galaxy/tools/parameters/validation.py

@@ -186,7 +186,9 @@ class LengthValidator(InRangeValidator):
         super().__init__(message, range_min=length_min, range_max=length_max, negate=negate)
 
     def validate(self, value, trans=None):
-        super().validate(len(value), trans)
+        if value is None:
+            raise ValueError("No value provided")
+        super().validate(len(value) if value else 0, trans)
 
 
 class DatasetOkValidator(Validator):

lib/galaxy_test/api/test_datasets.py

@@ -12,6 +12,7 @@ from galaxy.model.unittest_utils.store_fixtures import (
     one_hda_model_store_dict,
     TEST_SOURCE_URI,
 )
+from galaxy.tool_util.verify.test_data import TestDataResolver
 from galaxy.util.unittest_utils import skip_if_github_down
 from galaxy_test.base.api_asserts import assert_has_keys
 from galaxy_test.base.decorators import (
@@ -356,6 +357,15 @@ class TestDatasetsApi(ApiTestCase):
         self._assert_has_key(get_content_as_text_response.json(), "item_data")
         assert get_content_as_text_response.json().get("item_data") == contents
 
+    def test_get_content_as_text_with_compressed_text_data(self, history_id):
+        test_data_resolver = TestDataResolver()
+        with open(test_data_resolver.get_filename("1.fasta.gz"), mode="rb") as fh:
+            hda1 = self.dataset_populator.new_dataset(history_id, content=fh, ftype="fasta.gz", wait=True)
+        get_content_as_text_response = self._get(f"datasets/{hda1['id']}/get_content_as_text")
+        self._assert_status_code_is(get_content_as_text_response, 200)
+        self._assert_has_key(get_content_as_text_response.json(), "item_data")
+        assert ">hg17" in get_content_as_text_response.json().get("item_data")
+
     def test_anon_get_content_as_text(self, history_id):
         contents = "accessible data"
         hda1 = self.dataset_populator.new_dataset(history_id, content=contents, wait=True)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@galaxyproject/galaxy",
-  "version": "23.1.0",
+  "version": "24.0.0",
   "description": ".. figure:: https://galaxyproject.org/images/galaxy-logos/galaxy_project_logo.jpg    :alt: Galaxy Logo",
   "main": "index.js",
   "scripts": {
@@ -17,7 +17,7 @@
   },
   "homepage": "https://github.com/galaxyproject/galaxy#readme",
   "dependencies": {
-    "@galaxyproject/galaxy-client": "^23.1.1",
+    "@galaxyproject/galaxy-client": "^24.0.0",
     "cpy-cli": "^5.0.0"
   }
 }

test/unit/app/tools/test_parameter_validation.py

@@ -188,6 +188,14 @@ class TestParameterValidation(BaseParameterTestCase):
             p.validate("bar")
         p.validate("f")
         p.validate("foobarbaz")
+        p = self._parameter_for(
+            xml="""
+<param name="blah" type="text" optional="false">
+    <validator type="length" min="2" max="8"/>
+</param>"""
+        )
+        with self.assertRaisesRegex(ValueError, "No value provided"):
+            p.validate(None)
 
     def test_InRangeValidator(self):
         p = self._parameter_for(

test/unit/data/test_galaxy_mapping.py

@@ -881,6 +881,9 @@ class TestMappings(BaseModelTestCase):
         assert loaded_invocation
         assert loaded_invocation.history.id == history_id
 
+        # recover user after expunge
+        user = loaded_invocation.history.user
+
         step_1, step_2 = loaded_invocation.workflow.steps
 
         assert not step_1.subworkflow