diff --git a/client/src/components/Collections/common/CollectionEditView.vue b/client/src/components/Collections/common/CollectionEditView.vue new file mode 100644 index 00000000000..ca8297b010d --- /dev/null +++ b/client/src/components/Collections/common/CollectionEditView.vue @@ -0,0 +1,124 @@ + + + diff --git a/client/src/components/Collections/common/DatabaseEditTab.vue b/client/src/components/Collections/common/DatabaseEditTab.vue new file mode 100644 index 00000000000..dd9a9a5382f --- /dev/null +++ b/client/src/components/Collections/common/DatabaseEditTab.vue @@ -0,0 +1,59 @@ + + diff --git a/client/src/components/Collections/common/SuitableConvertersTab.vue b/client/src/components/Collections/common/SuitableConvertersTab.vue new file mode 100644 index 00000000000..e31ad5ca46f --- /dev/null +++ b/client/src/components/Collections/common/SuitableConvertersTab.vue @@ -0,0 +1,51 @@ + + + diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue b/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue index 83ca28f9987..644e22e0332 100644 --- a/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue +++ b/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue @@ -1,45 +1,87 @@ + diff --git a/client/src/components/providers/index.js b/client/src/components/providers/index.js index 48ba104c87d..eccc245799e 100644 --- a/client/src/components/providers/index.js +++ b/client/src/components/providers/index.js @@ -2,6 +2,8 @@ export { // DatasetProvider, DatasetCollectionContentProvider, + GenomeProvider, + SuitableConvertersProvider, JobProvider, } from "./storeProviders"; diff --git a/client/src/components/providers/storeProviders.js b/client/src/components/providers/storeProviders.js index aacaf31cec4..ddf74c81608 100644 --- a/client/src/components/providers/storeProviders.js +++ b/client/src/components/providers/storeProviders.js @@ -80,6 +80,38 @@ export const DatasetCollectionProvider = { }, }; +export const GenomeProvider = { + mixins: [SimpleProviderMixin], + async mounted() { + await this.load(); + }, + methods: { + ...mapCacheActions(["fetchUploadGenomes"]), + async load() { + this.loading = true; + let genomes = this.getUploadGenomes(); + if (genomes == null || genomes.length == 0) { + await this.fetchUploadGenomes(); + genomes = this.getUploadGenomes(); + } + this.item = genomes; + this.loading = false; + }, + }, + computed: { + ...mapGetters(["getUploadGenomes"]), + }, +}; + +export const SuitableConvertersProvider = { + mixins: [SimpleProviderMixin], + computed: { + url() { + return prependPath(`/api/dataset_collections/${this.id}/suitable_converters`); + }, + }, +}; + export const DatasetCollectionContentProvider = { mixins: [SimpleProviderMixin], computed: { diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js index 04caf52a7df..5a527c67e08 100644 --- a/client/src/entry/analysis/AnalysisRouter.js +++ b/client/src/entry/analysis/AnalysisRouter.js @@ -31,6 +31,7 @@ import TrsImport from "components/Workflow/TrsImport.vue"; import TrsSearch from "components/Workflow/TrsSearch.vue"; import InteractiveTools from "components/InteractiveTools/InteractiveTools.vue"; import WorkflowList from "components/Workflow/WorkflowList.vue"; +import CollectionEditView from "components/Collections/common/CollectionEditView.vue"; import HistoryImport from "components/HistoryImport.vue"; import { HistoryExport } from "components/HistoryExport/index"; import HistoryView from "components/HistoryView.vue"; @@ -98,6 +99,7 @@ export const getAnalysisRouter = (Galaxy) => { "(/)jobs(/)(:job_id)(/)view": "show_job", "(/)custom_builds": "show_custom_builds", "(/)datasets/edit": "show_dataset_edit_attributes", + "(/)collection(/)edit(/)(:collection_id)": "show_collection_edit_attributes", "(/)datasets/error": "show_dataset_error", "(/)datasets(/)(:dataset_id)/details": "show_dataset_details", "(/)interactivetool_entry_points(/)list": "show_interactivetool_list", @@ -388,6 +390,10 @@ export const getAnalysisRouter = (Galaxy) => { } }, + show_collection_edit_attributes: function (collection_id) { + this._display_vue_helper(CollectionEditView, { collection_id: collection_id }); + }, + show_dataset_error: function (params) { const datasetId = params.dataset_id; this._display_vue_helper(DatasetError, { datasetId: datasetId }); diff --git a/client/src/store/collectionAttributesStore.js b/client/src/store/collectionAttributesStore.js new file mode 100644 index 00000000000..5cae14d6144 --- /dev/null +++ b/client/src/store/collectionAttributesStore.js @@ -0,0 +1,32 @@ +export const state = { + collectionAttributes: {}, +}; +import Vue from "vue"; +import { prependPath } from "utils/redirect"; +import axios from "axios"; + +const getters = { + getCollectionAttributes: (state) => (collectionId) => { + return state.collectionAttributes[collectionId] || null; + }, +}; + +const actions = { + fetchCollectionAttributes: async ({ commit }, collectionId) => { + const { data } = await axios.get(prependPath("api/dataset_collections/" + collectionId + "/attributes")); + commit("saveCollectionAttributes", { collectionId, collectionAttributes: data }); + }, +}; + +const mutations = { + saveCollectionAttributes: (state, { collectionId, collectionAttributes }) => { + Vue.set(state.collectionAttributes, collectionId, collectionAttributes); + }, +}; + +export const collectionAttributesStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/src/store/datatypeStore.js b/client/src/store/datatypeStore.js new file mode 100644 index 00000000000..85dadbd4e3d --- /dev/null +++ b/client/src/store/datatypeStore.js @@ -0,0 +1,34 @@ +export const state = { + uploadDatatypes: [], +}; +import UploadUtils from "mvc/upload/upload-utils"; + +const getters = { + getUploadDatatypes: (state) => () => { + return state.uploadDatatypes; + }, +}; + +const actions = { + fetchUploadDatatypes: async ({ commit }) => { + try { + const data = await UploadUtils.getUploadDatatypes(false, UploadUtils.AUTO_EXTENSION); + commit("saveUploadDatatypes", { datatypes: data }); + } catch (err) { + console.log("Error: unable to load datatypes", err); + } + }, +}; + +const mutations = { + saveUploadDatatypes: (state, { datatypes }) => { + state.uploadDatatypes = datatypes; + }, +}; + +export const datatypeStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/src/store/genomeStore.js b/client/src/store/genomeStore.js new file mode 100644 index 00000000000..8c4c064d07a --- /dev/null +++ b/client/src/store/genomeStore.js @@ -0,0 +1,34 @@ +export const state = { + uploadGenomes: [], +}; +import UploadUtils from "mvc/upload/upload-utils"; + +const getters = { + getUploadGenomes: (state) => () => { + return state.uploadGenomes; + }, +}; + +const actions = { + fetchUploadGenomes: async ({ commit }) => { + try { + const data = await UploadUtils.getUploadGenomes(UploadUtils.DEFAULT_GENOME); + commit("saveUploadGenomes", { genomes: data }); + } catch (err) { + console.log("Error: unable to load genomes", err); + } + }, +}; + +const mutations = { + saveUploadGenomes: (state, { genomes }) => { + state.uploadGenomes = genomes; + }, +}; + +export const genomeStore = { + state, + getters, + actions, + mutations, +}; diff --git a/client/src/store/index.js b/client/src/store/index.js index 30685be19cd..b6b06ed2963 100644 --- a/client/src/store/index.js +++ b/client/src/store/index.js @@ -22,6 +22,9 @@ import { toolStore } from "./toolStore"; import { datasetPathDestinationStore } from "./datasetPathDestinationStore"; import { datasetExtFilesStore } from "./datasetExtFilesStore"; import { jobStore } from "./jobStore"; +import { collectionAttributesStore } from "./collectionAttributesStore"; +import { genomeStore } from "./genomeStore"; +import { datatypeStore } from "./datatypeStore"; import { panelStore } from "./panelStore"; // beta features @@ -57,6 +60,9 @@ export function createStore() { workflows: workflowStore, informationStore: jobStore, tools: toolStore, + collectionAttributesStore: collectionAttributesStore, + genomeStore: genomeStore, + datatypeStore: datatypeStore, }, }; diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index b3924e2762b..4708ae1e16e 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -76,6 +76,25 @@ def validate(dataset_instance): return datatype_validation +def get_params_and_input_name(converter, deps, target_context=None): + # Generate parameter dictionary + params = {} + # determine input parameter name and add to params + input_name = 'input1' + for key, value in converter.inputs.items(): + if deps and value.name in deps: + params[value.name] = deps[value.name] + elif value.type == 'data': + input_name = key + + # add potentially required/common internal tool parameters e.g. '__job_resource' + if target_context: + for key, value in target_context.items(): + if key.startswith('__'): + params[key] = value + return params, input_name + + class DataMeta(abc.ABCMeta): """ Metaclass for Data class. Sets up metadata spec. @@ -620,20 +639,9 @@ class Data(metaclass=DataMeta): if converter is None: raise Exception(f"A converter does not exist for {original_dataset.ext} to {target_type}.") - # Generate parameter dictionary - params = {} - # determine input parameter name and add to params - input_name = 'input1' - for key, value in converter.inputs.items(): - if deps and value.name in deps: - params[value.name] = deps[value.name] - elif value.type == 'data': - input_name = key - # add potentially required/common internal tool parameters e.g. '__job_resource' - if target_context: - for key, value in target_context.items(): - if key.startswith('__'): - params[key] = value + + params, input_name = get_params_and_input_name(converter, deps, target_context) + params[input_name] = original_dataset # Make the target datatype available to the converter params['__target_datatype__'] = target_type diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index ebfe09185f9..d9d0f4cbd42 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -3,6 +3,7 @@ import logging from sqlalchemy.orm import joinedload, Query from galaxy import model +from galaxy.datatypes.registry import Registry from galaxy.exceptions import ( ItemAccessibilityException, MessageException, @@ -218,6 +219,38 @@ class DatasetCollectionManager: dataset_collection.collection_type = collection_type return dataset_collection + def get_converters_for_collection(self, trans, id, datatypes_registry: Registry, instance_type="history"): + dataset_collection_instance = self.get_dataset_collection_instance( + trans, + id=id, + instance_type=instance_type, + check_ownership=True + ) + dbkeys_and_extensions = dataset_collection_instance.dataset_dbkeys_and_extensions_summary + suitable_converters = set() + first_extension = True + most_recent_datatype = None + # TODO error checking + for datatype in dbkeys_and_extensions[1]: + new_converters = datatypes_registry.get_converters_by_datatype(datatype) + set_of_new_converters = set() + for tgt_type, tgt_val in new_converters.items(): + converter = (tgt_type, tgt_val) + set_of_new_converters.add(converter) + if (first_extension is True): + suitable_converters = set_of_new_converters + most_recent_datatype = datatype + first_extension = False + else: + suitable_converters = suitable_converters.intersection(set_of_new_converters) + if suitable_converters: + most_recent_datatype = datatype + suitable_tool_ids = list() + for tool in suitable_converters: + tool_info = {"tool_id": tool[1].id, "name": tool[1].name, "target_type": tool[0], "original_type": most_recent_datatype} + suitable_tool_ids.append(tool_info) + return suitable_tool_ids + def _element_identifiers_to_elements(self, trans, collection_type_description, diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index d810fcc9757..036f0353d58 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -5,6 +5,7 @@ HistoryDatasetCollectionAssociations (HDCAs) are datasets contained or created i history. """ import logging +from typing import Dict from galaxy import model from galaxy.managers import ( @@ -39,6 +40,11 @@ def stream_dataset_collection(dataset_collection_instance, upstream_mod_zip=Fals return archive +def set_collection_attributes(dataset_element, *payload): + for attribute, value in payload: + setattr(dataset_element, attribute[1], value[1]) + + # TODO: to DatasetCollectionInstanceManager class HDCAManager( base.ModelManager, @@ -77,7 +83,9 @@ class HDCAManager( returned.append(processed) return returned - # TODO: un-stub + def update_attributes(self, content, payload: Dict): + # pre-requisite checked that attributes are valid + self.map_datasets(content, fn=lambda item, *args: set_collection_attributes(item, payload.items())) # serializers diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 82245aedfa8..72523b989c5 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5530,6 +5530,7 @@ class HistoryDatasetCollectionAssociation( == HistoryDatasetCollectionAssociation.id), # type: ignore back_populates='hidden_beneath_collection_instance') + dict_dbkeysandextensions_visible_keys = ['dbkeys', 'extensions'] editable_keys = ('name', 'deleted', 'visible') def __init__(self, deleted=False, visible=True, **kwd): @@ -5566,6 +5567,24 @@ class HistoryDatasetCollectionAssociation( else: return None + @property + def dataset_dbkeys_and_extensions_summary(self): + if not hasattr(self, '_dataset_dbkeys_and_extensions_summary'): + rows = self.collection._get_nested_collection_attributes(hda_attributes=('_metadata', 'extension')) + extensions = set() + dbkeys = set() + for row in rows: + if row is not None: + dbkey_field = row._metadata.get('dbkey') + if isinstance(dbkey_field, list): + for dbkey in dbkey_field: + dbkeys.add(dbkey) + else: + dbkeys.add(dbkey_field) + extensions.add(row.extension) + self._dataset_dbkeys_and_extensions_summary = (dbkeys, extensions) + return self._dataset_dbkeys_and_extensions_summary + @property def job_source_id(self): return self.implicit_collection_jobs_id or self.job_id @@ -5620,16 +5639,23 @@ class HistoryDatasetCollectionAssociation( def to_dict(self, view='collection'): original_dict_value = super().to_dict(view=view) - dict_value = dict( - hid=self.hid, - history_id=self.history.id, - history_content_type=self.history_content_type, - visible=self.visible, - deleted=self.deleted, - job_source_id=self.job_source_id, - job_source_type=self.job_source_type, - **self._base_to_dict(view=view) - ) + if (view == 'dbkeysandextensions'): + (dbkeys, extensions) = self.dataset_dbkeys_and_extensions_summary + dict_value = dict( + dbkey=dbkeys.pop() if len(dbkeys) == 1 else "?", + extension=extensions.pop() if len(extensions) == 1 else "auto" + ) + else: + dict_value = dict( + hid=self.hid, + history_id=self.history.id, + history_content_type=self.history_content_type, + visible=self.visible, + deleted=self.deleted, + job_source_id=self.job_source_id, + job_source_type=self.job_source_type, + **self._base_to_dict(view=view) + ) dict_value.update(original_dict_value) diff --git a/lib/galaxy/selenium/navigation.yml b/lib/galaxy/selenium/navigation.yml index 6b019cf6a12..d768e1c741d 100644 --- a/lib/galaxy/selenium/navigation.yml +++ b/lib/galaxy/selenium/navigation.yml @@ -269,6 +269,14 @@ history_panel: options_use_beta_history: type: xpath selector: '//a[text()="Use Beta History Panel"]' + options_use_legacy_history: + type: xpath + selector: '//button[contains(span, "Return to legacy history panel")]' + + collection_menu_button: '.collection-menu' + collection_menu_edit_attributes: + type: xpath + selector: '//button[@title="Edit attributes"]' new_history_button: '.history-new-button' multi_view_button: '.history-view-multi-button' histories_operation_menu: '.histories-operation-menu' @@ -306,6 +314,16 @@ edit_dataset_attributes: type: xpath selector: '//li[normalize-space() = "${dbkey_text}"]' +edit_collection_attributes: + selectors: + database_genome_tab: + type: xpath + selector: '//a[contains(text(), "Database/Build")]' + database_value: + type: xpath + selector: '//span[contains(text(), "${dbkey}")]' + save_btn: '.save-collection-edit' + tool_panel: diff --git a/lib/galaxy/webapps/galaxy/api/dataset_collections.py b/lib/galaxy/webapps/galaxy/api/dataset_collections.py index d91cc67ad00..be12e008075 100644 --- a/lib/galaxy/webapps/galaxy/api/dataset_collections.py +++ b/lib/galaxy/webapps/galaxy/api/dataset_collections.py @@ -3,14 +3,16 @@ from logging import getLogger import routes from galaxy import exceptions +from galaxy.datatypes.registry import Registry from galaxy.managers.base import decode_id from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.collections_util import ( api_payload_to_create_params, dictify_dataset_collection_instance, - dictify_element_reference + dictify_element_reference, ) from galaxy.managers.context import ProvidesHistoryContext +from galaxy.managers.hdcas import HDCAManager from galaxy.managers.histories import HistoryManager from galaxy.web import expose_api from galaxy.webapps.base.controller import UsesLibraryMixinItems @@ -24,6 +26,9 @@ class DatasetCollectionsController( UsesLibraryMixinItems, ): history_manager: HistoryManager = depends(HistoryManager) + hdca_manager: HDCAManager = depends(HDCAManager) + collection_manager: DatasetCollectionManager = depends(DatasetCollectionManager) + datatypes_registry: Registry = depends(Registry) @expose_api def index(self, trans, **kwd): @@ -64,6 +69,48 @@ class DatasetCollectionsController( return dictify_dataset_collection_instance(dataset_collection_instance, security=trans.security, parent=create_params["parent"]) + @expose_api + def update(self, trans: ProvidesHistoryContext, payload: dict, id): + """ + Iterate over all datasets of a collection and copy datasets with new attributes to a new collection. + e.g attributes = {'dbkey': 'dm3'} + + * PUT /api/dataset_collections/{hdca_id}: + create a new dataset collection instance. + """ + + if len(payload) != 1: + raise exceptions.RequestParameterInvalidException("Update one attribute at a time.") + if 'dbkey' not in payload: + raise exceptions.RequestParameterInvalidException("This attribute cannot be modified.") + + self.collection_manager.copy(trans, trans.history, "hdca", id, copy_elements=True, dataset_instance_attributes=payload) + trans.sa_session.flush() + + @expose_api + def attributes(self, trans: ProvidesHistoryContext, id, instance_type='history'): + """ + GET /api/dataset_collections/{hdca_id}/attributes + + Returns dbkey/extension for collection elements + """ + dataset_collection_instance = self.__service.get_dataset_collection_instance( + trans, + id=id, + instance_type=instance_type, + check_ownership=True + ) + return dataset_collection_instance.to_dict(view="dbkeysandextensions") + + @expose_api + def suitable_converters(self, trans: ProvidesHistoryContext, id, instance_type='history', **kwds): + """ + GET /api/dataset_collections/{hdca_id}/suitable_converters + + Returns suitable converters for all datatypes in collection + """ + return self.collection_manager.get_converters_for_collection(trans, id, self.datatypes_registry, instance_type) + @expose_api def show(self, trans: ProvidesHistoryContext, id, instance_type='history', **kwds): """ diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 6c05817506b..a58ed8e22cf 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -4,6 +4,8 @@ from json import dumps, loads from typing import Any, cast, Dict, Optional from galaxy import exceptions, util, web +from galaxy.datatypes.data import get_params_and_input_name +from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.collections_util import dictify_dataset_collection_instance from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager @@ -36,6 +38,7 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): """ history_manager: HistoryManager = depends(HistoryManager) hda_manager: HDAManager = depends(HDAManager) + hdca_manager: DatasetCollectionManager = depends(DatasetCollectionManager) @expose_api_anonymous_and_sessionless def index(self, trans: GalaxyWebTransaction, **kwds): @@ -426,6 +429,51 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): rval.append(citation.to_dict('bibtex')) return rval + @expose_api + def conversion(self, trans: GalaxyWebTransaction, tool_id, payload, **kwd): + converter = self._get_tool(tool_id, user=trans.user) + target_type = payload.get("target_type") + source_type = payload.get("source_type") + input_src = payload.get("src") + input_id = payload.get("id") + # List of string of dependencies + try: + deps = trans.app.datatypes_registry.converter_deps[source_type][target_type] + except KeyError: + deps = {} + # Generate parameter dictionary + params, input_name = get_params_and_input_name(converter, deps) + params = {} + # determine input parameter name and add to params + + params[input_name] = { + "values": [ + { + "id": input_id, + "src": input_src, + } + ], + "batch": input_src == "hdca", + } + history_id = payload.get('history_id') + if history_id: + decoded_id = self.decode_id(history_id) + target_history = self.history_manager.get_owned(decoded_id, trans.user, current_history=trans.history) + else: + if input_src == "hdca": + target_history = self.hdca_manager.get_dataset_collection_instance(trans, instance_type='history', id=input_id).history + elif input_src == "hda": + decoded_id = trans.app.security.decode_id(input_id) + target_history = self.hda_manager.get_accessible(decoded_id, trans.user).history + self.history_manager.error_unless_owner(target_history, trans.user, current_history=trans.history) + else: + raise exceptions.RequestParameterInvalidException("Must run conversion on either hdca or hda.") + + # Make the target datatype available to the converter + params['__target_datatype__'] = target_type + vars = converter.handle_input(trans, params, history=target_history) + return self._handle_inputs_output_to_api_response(trans, converter, target_history, vars) + @expose_api_anonymous_and_sessionless def xrefs(self, trans: GalaxyWebTransaction, id, **kwds): tool = self._get_tool(id, user=trans.user) @@ -568,6 +616,24 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): vars = tool.handle_input(trans, incoming, history=target_history, use_cached_job=use_cached_job, input_format=input_format) + new_pja_flush = False + for job in vars.get('jobs', []): + if inputs.get('send_email_notification', False): + # Unless an anonymous user is invoking this via the API it + # should never be an option, but check and enforce that here + if trans.user is None: + raise exceptions.ToolExecutionError("Anonymously run jobs cannot send an email notification.") + else: + job_email_action = PostJobAction('EmailAction') + job.add_post_job_action(job_email_action) + new_pja_flush = True + + if new_pja_flush: + trans.sa_session.flush() + + return self._handle_inputs_output_to_api_response(trans, tool, target_history, vars) + + def _handle_inputs_output_to_api_response(self, trans, tool, target_history, vars): # TODO: check for errors and ensure that output dataset(s) are available. output_datasets = vars.get('out_data', []) rval: Dict[str, Any] = {'outputs': [], 'output_collections': [], 'jobs': [], 'implicit_collections': []} @@ -587,21 +653,8 @@ class ToolsController(BaseGalaxyAPIController, UsesVisualizationMixin): output_dict['output_name'] = output_name outputs.append(trans.security.encode_dict_ids(output_dict, skip_startswith="metadata_")) - new_pja_flush = False for job in vars.get('jobs', []): rval['jobs'].append(self.encode_all_ids(trans, job.to_dict(view='collection'), recursive=True)) - if inputs.get('send_email_notification', False): - # Unless an anonymous user is invoking this via the API it - # should never be an option, but check and enforce that here - if trans.user is None: - raise exceptions.ToolExecutionError("Anonymously run jobs cannot send an email notification.") - else: - job_email_action = PostJobAction('EmailAction') - job.add_post_job_action(job_email_action) - new_pja_flush = True - - if new_pja_flush: - trans.sa_session.flush() for output_name, collection_instance in vars.get('output_collections', []): history = target_history or trans.history diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 4ce17e38514..710c1557ab4 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -178,6 +178,7 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs): webapp.add_client_route('/histories/show_structure') webapp.add_client_route('/datasets/list') webapp.add_client_route('/datasets/edit') + webapp.add_client_route('/collection/edit/{collection_id}') webapp.add_client_route('/datasets/error') webapp.add_client_route('/jobs/{job_id}/view') webapp.add_client_route('/datasets/{dataset_id}/details') @@ -425,6 +426,7 @@ def populate_api_routes(webapp, app): webapp.mapper.connect('/api/tools/{id:.+?}/test_data', action='test_data', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/citations', action='citations', controller="tools") + webapp.mapper.connect('/api/tools/{tool_id:.+?}/convert', action='conversion', controller="tools", conditions=dict(method=["POST"])) webapp.mapper.connect('/api/tools/{id:.+?}/xrefs', action='xrefs', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/download', action='download', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/requirements', action='requirements', controller="tools") @@ -550,6 +552,22 @@ def populate_api_routes(webapp, app): action='download_dataset_collection', conditions=dict(method=["GET"])) + webapp.mapper.connect("/api/dataset_collections/{id}", + controller='dataset_collections', + action='update', + conditions=dict(method=["PUT"])) + + webapp.mapper.connect("/api/dataset_collections/{id}/attributes", + controller='dataset_collections', + action='attributes', + conditions=dict(method=["GET"])) + + webapp.mapper.connect("api_suitable_converters", + "/api/dataset_collections/{id}/suitable_converters", + controller='dataset_collections', + action='suitable_converters', + conditions=dict(method=['GET'])) + webapp.mapper.connect("/api/histories/{history_id}/jobs_summary", action="index_jobs_summary", controller='history_contents', diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py index dedc6a00b56..ace5cdb0661 100644 --- a/lib/galaxy_test/api/test_dataset_collections.py +++ b/lib/galaxy_test/api/test_dataset_collections.py @@ -356,6 +356,94 @@ class DatasetCollectionApiTestCase(ApiTestCase): assert len(offset_contents) == 1 assert offset_contents[0]['element_index'] == 1 + def test_get_suitable_converters_single_datatype(self): + response = self.dataset_collection_populator.upload_collection(self.history_id, "list:paired", elements=[ + { + "name": "test0", + "elements": [ + {"src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "bed"}, + {"src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "bed"}, + ] + }, + { + "name": "test1", + "elements": [ + {"src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "bed"}, + {"src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "bed"}, + ] + } + ]) + self._assert_status_code_is(response, 200) + hdca_list_id = response.json()["outputs"][0]["id"] + converters = self._get("dataset_collections/" + hdca_list_id + "/suitable_converters") + expected = [ + 'CONVERTER_bed_to_fli_0', + 'CONVERTER_interval_to_bed_0', + 'CONVERTER_bed_gff_or_vcf_to_bigwig_0', + 'CONVERTER_bed_to_gff_0', + 'CONVERTER_interval_to_bgzip_0', + 'tabular_to_csv', + 'CONVERTER_interval_to_bed6_0', + 'CONVERTER_interval_to_bedstrict_0', + 'CONVERTER_interval_to_tabix_0', + 'CONVERTER_interval_to_bed12_0'] + actual = [] + for converter in converters.json(): + actual.append(converter["tool_id"]) + assert sorted(actual) == sorted(expected) + + def test_get_suitable_converters_different_datatypes_matches(self): + response = self.dataset_collection_populator.upload_collection(self.history_id, "list:paired", elements=[ + { + "name": "test0", + "elements": [ + {"src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "bed"}, + {"src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "bed"}, + ] + }, + { + "name": "test1", + "elements": [ + {"src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "tabular"}, + {"src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "tabular"}, + ] + } + ]) + self._assert_status_code_is(response, 200) + hdca_list_id = response.json()["outputs"][0]["id"] + converters = self._get("dataset_collections/" + hdca_list_id + "/suitable_converters") + expected = ['tabular_to_csv'] + actual = [] + for converter in converters.json(): + actual.append(converter["tool_id"]) + assert sorted(actual) == sorted(expected) + + def test_get_suitable_converters_different_datatypes_no_matches(self): + response = self.dataset_collection_populator.upload_collection(self.history_id, "list:paired", elements=[ + { + "name": "test0", + "elements": [ + {"src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "bed"}, + {"src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "bed"}, + ] + }, + { + "name": "test1", + "elements": [ + {"src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "fasta"}, + {"src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "fasta"}, + ] + } + ]) + self._assert_status_code_is(response, 200) + hdca_list_id = response.json()["outputs"][0]["id"] + converters = self._get("dataset_collections/" + hdca_list_id + "/suitable_converters") + expected = [] + actual = [] + for converter in converters.json(): + actual.append(converter["tool_id"]) + assert sorted(actual) == sorted(expected) + def test_collection_tools_tag_propagation(self): elements = [{"src": "files", "tags": ["name:element_tag"]}] targets = [{ diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 1d583c4d6da..28e9953a4f2 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -433,6 +433,82 @@ class ToolsTestCase(ApiTestCase, TestsTools): expected_names = {'velveth_test1/Roadmaps', 'velveth_test1/output.html', 'velveth_test1/Sequences', 'velveth_test1/Log', 'velveth_test1/output/', 'velveth_test1/output/1'} assert set(namelist) == expected_names + @uses_test_history(require_new=False) + def test_convert_dataset_explicit_history(self, history_id): + fasta1_contents = open(self.get_filename("1.fasta")).read() + hda1 = self.dataset_populator.new_dataset(history_id, content=fasta1_contents) + + payload = { + "src": "hda", + "id": hda1["id"], + "source_type": "fasta", + "target_type": "tabular", + "history_id": history_id + } + create_response = self._post("tools/CONVERTER_fasta_to_tabular/convert", data=payload) + self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True) + create_response.raise_for_status() + assert len(create_response.json()["implicit_collections"]) == 0 + for output in create_response.json()["outputs"]: + assert output["file_ext"] == "tabular" + + @uses_test_history(require_new=False) + def test_convert_dataset_implicit_history(self, history_id): + fasta1_contents = open(self.get_filename("1.fasta")).read() + hda1 = self.dataset_populator.new_dataset(history_id, content=fasta1_contents) + + payload = { + "src": "hda", + "id": hda1["id"], + "source_type": "fasta", + "target_type": "tabular" + } + create_response = self._post("tools/CONVERTER_fasta_to_tabular/convert", data=payload) + self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True) + create_response.raise_for_status() + assert len(create_response.json()["implicit_collections"]) == 0 + for output in create_response.json()["outputs"]: + assert output["file_ext"] == "tabular" + + @uses_test_history(require_new=False) + def test_convert_hdca(self, history_id): + data = [ + { + "name": "test0", + "elements": [ + {"src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "fasta"}, + {"src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "fasta"}, + ] + }, + { + "name": "test1", + "elements": [ + {"src": "pasted", "paste_content": "789\n", "name": "forward", "ext": "fasta"}, + {"src": "pasted", "paste_content": "0ab\n", "name": "reverse", "ext": "fasta"}, + ] + } + ] + hdca1 = self.dataset_collection_populator.upload_collection(history_id, "list:paired", elements=data) + self._assert_status_code_is(hdca1, 200) + + payload = { + "src": "hdca", + "id": hdca1.json()["outputs"][0]["id"], + "source_type": "fasta", + "target_type": "tabular", + "history_id": history_id + } + create_response = self._post("tools/CONVERTER_fasta_to_tabular/convert", payload) + + self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True) + create_response.raise_for_status() + + assert create_response.json()["implicit_collections"] != [] + hdca_id = create_response.json()["implicit_collections"][0]["hid"] + fetchedResponse = self.dataset_populator.get_history_collection_details(history_id, hid=hdca_id) + for element in fetchedResponse["elements"][0]["object"]["elements"]: + assert element["object"]["file_ext"] == "tabular" + def test_unzip_collection(self): with self.dataset_populator.test_history() as history_id: hdca_id = self._build_pair(history_id, ["123", "456"]) diff --git a/lib/galaxy_test/selenium/test_collection_edit.py b/lib/galaxy_test/selenium/test_collection_edit.py new file mode 100644 index 00000000000..bceb202744e --- /dev/null +++ b/lib/galaxy_test/selenium/test_collection_edit.py @@ -0,0 +1,74 @@ +from selenium.webdriver.common.keys import Keys + +from .framework import ( + selenium_test, + SeleniumTestCase +) + + +class CollectionEditTestCase(SeleniumTestCase): + + ensure_registered = True + + @selenium_test + def test_change_dbkey_simple_list(self): + self.use_beta_history() + self.create_simple_list_collection() + self.open_collection_edit_view() + self.navigate_to_database_tab() + dbkeyValue = "Additional" + self.check_current_dbkey_value(dbkeyValue) + dbkeyNew = "hg17" + self.change_dbkey_value_and_click_submit(dbkeyValue, dbkeyNew) + self.history_panel_wait_for_hid_ok(4) + self.use_beta_history() + self.open_collection_edit_view() + self.navigate_to_database_tab() + self.check_current_dbkey_value(dbkeyNew) + + def create_simple_list_collection(self): + self.perform_upload(self.get_filename("1.fasta")) + self._wait_for_and_select([1]) + + self._collection_dropdown("build list") + + self.collection_builder_set_name("my cool list") + self.screenshot("collection_builder_list") + self.collection_builder_create() + self._wait_for_hid_visible(2) + + def open_collection_edit_view(self): + self.components.history_panel.collection_menu_edit_attributes.wait_for_and_click() + + def navigate_to_database_tab(self): + self.components.edit_collection_attributes.database_genome_tab.wait_for_and_click() + + def check_current_dbkey_value(self, dbkeyValue): + self.components.edit_collection_attributes.database_value(dbkey=dbkeyValue).wait_for_visible() + + def change_dbkey_value_and_click_submit(self, dbkeyValue, dbkeyNew): + self.components.edit_collection_attributes.database_value(dbkey=dbkeyValue).wait_for_and_click() + self.driver.find_element_by_css_selector("input.multiselect__input").send_keys(dbkeyNew) + self.driver.find_element_by_css_selector("input.multiselect__input").send_keys(Keys.ENTER) + self.components.edit_collection_attributes.save_btn.wait_for_and_click() + + def _wait_for_and_select(self, hids): + """ + Waits for uploads to pass through queued, running, ok. Not all the states are not guaranteed + depending on how fast the upload goes compared to the history polling updates, it might just + skip to the end for a really fast upload + """ + + for hid in hids: + timeout = self.wait_length(self.wait_types.JOB_COMPLETION) + row_selector = self.content_item_by_attributes(hid=hid, state="ok") + row = self.wait_for_present(row_selector, timeout=timeout) + row.send_keys(" ") + + def _collection_dropdown(self, option_description): + return self.use_bootstrap_dropdown(option=option_description, menu="new content menu") + + def _wait_for_hid_visible(self, hid, state="ok"): + timeout = self.wait_length(self.wait_types.JOB_COMPLETION) + row_selector = self.content_item_by_attributes(hid=hid, state=state) + self.wait_for_visible(row_selector, timeout=timeout) diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index a5b347c9c55..2bab0612eb3 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -417,6 +417,21 @@ class MappingTests(BaseModelTestCase): assert c4.dataset_elements == [dce1, dce2] assert c4.element_identifiers_extensions_and_paths == [(('outer_list', 'inner_list', 'forward'), 'bam', 'mock_dataset_14.dat'), (('outer_list', 'inner_list', 'reverse'), 'txt', 'mock_dataset_14.dat')] + def test_dataset_dbkeys_and_extensions_summary(self): + model = self.model + u = model.User(email="mary2@example.com", password="password") + h1 = model.History(name="History 1", user=u) + d1 = model.HistoryDatasetAssociation(extension="bam", dbkey="hg19", history=h1, create_dataset=True, sa_session=model.session) + d2 = model.HistoryDatasetAssociation(extension="txt", dbkey="hg19", history=h1, create_dataset=True, sa_session=model.session) + c1 = model.DatasetCollection(collection_type='paired') + dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) + dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) + hdca = model.HistoryDatasetCollectionAssociation(collection=c1, history=h1) + model.session.add_all([d1, d2, c1, dce1, dce2, hdca]) + model.session.flush() + assert hdca.dataset_dbkeys_and_extensions_summary[0] == {"hg19"} + assert hdca.dataset_dbkeys_and_extensions_summary[1] == {"bam", "txt"} + def test_default_disk_usage(self): model = self.model