From 08eb9859f25dff98edbf2fc4be95012219d389a5 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 13 Nov 2017 16:29:31 -0500 Subject: [PATCH 1/9] enhance hdca creation endpoint to accept lddas by implementing automatic conversion of any such elements --- .../webapps/galaxy/api/history_contents.py | 61 ++++++++++++++++--- 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 7f027c7af01..6af7b365101 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -299,13 +299,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary # copy from library dataset hda = None if source == 'library': - ld = self.get_library_dataset(trans, content, check_ownership=False, check_accessible=False) - # TODO: why would get_library_dataset NOT return a library dataset? - if type(ld) is not trans.app.model.LibraryDataset: - raise exceptions.RequestParameterInvalidException( - "Library content id ( %s ) is not a dataset" % content) - # insert into history - hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True) + hda = self.__create_hda_from_ldda(trans, content, history, check_ownership=False, check_accessible=False) # copy an existing, accessible hda elif source == 'hda': @@ -321,6 +315,15 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary return self.hda_serializer.serialize_to_view(hda, user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed')) + def __create_hda_from_ldda(self, trans, content, history, check_ownership, check_accessible): + hda = None + ld = self.get_library_dataset(trans, content, check_ownership=False, check_accessible=False) + if type(ld) is not trans.app.model.LibraryDataset: + raise exceptions.RequestParameterInvalidException( + "Library content id ( %s ) is not a dataset" % content) + hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True) + return hda + def __create_datasets_from_library_folder(self, trans, history, payload, **kwd): rval = [] @@ -366,10 +369,54 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary return rval def __create_dataset_collection(self, trans, history, payload, **kwd): + """Create hdca in a history from the list of element identifiers + + :param history: history the new hdca should be added to + :type history: History + :param source: whether to create a new collection or copy existing one + :type source: str + :param payload: dictionary structure containing: + :param collection_type: type (and depth) of the new collection + :type name: str + :param element_identifiers: list of elements that should be in the new collection + :param element: one member of the collection + :param name: name of the element + :type name: str + :param src: source of the element (hda/ldda) + :type src: str + :param id: identifier + :type id: str + :type element: dict + :type name: list + :param name: name of the collection + :type name: str + :param hide_source_items: whether to mark the original hdas as hidden + :type name: bool + :type payload: dict + + .. note:: Elements may be nested depending on the collection_type + + :returns: dataset collection information + :rtype: dict + + :raises: RequestParameterInvalidException, RequestParameterMissingException + """ source = kwd.get("source", payload.get("source", "new_collection")) service = trans.app.dataset_collections_service if source == "new_collection": create_params = api_payload_to_create_params(payload) + converted_identifiers = [] + changed = False + for ei in payload.get('element_identifiers'): + # Convert lddas to hdas since there is no direct representation of library items in history. + if ei['src'] == 'ldda': + hda = self.__create_hda_from_ldda(trans, ei['id'], history, check_ownership=False, check_accessible=False) + converted_identifiers.append({"name": ei["name"], "src": "hda", "id": trans.security.encode_id(hda.id)}) + changed = True + else: + converted_identifiers.append(ei) + if changed: + create_params['element_identifiers'] = converted_identifiers dataset_collection_instance = service.create( trans, parent=history, From 5609bfc307cd9a6bdd86cd49128d3c4f8407613b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 14 Nov 2017 09:22:56 -0500 Subject: [PATCH 2/9] More API library testing. --- test/api/test_libraries.py | 22 ++++++++-------------- test/base/populators.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index b89019d106b..0029fb04bdc 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -4,7 +4,6 @@ from base.populators import ( DatasetPopulator, LibraryPopulator, TestsDatasets, - wait_on_state ) @@ -83,20 +82,15 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): create_response = self._post("folders/%s/contents" % folder_id, payload) self._assert_status_code_is(create_response, 403) + def test_show_private_dataset_permissions(self): + library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True) + with self._different_user(): + response = self.library_populator.show_ldda(library["id"], library_dataset["id"]) + # TODO: this should really be 403 and a proper JSON exception. + self._assert_status_code_is(response.status_code, 400) + def test_create_dataset(self): - library = self.library_populator.new_private_library("ForCreateDatasets") - payload, files = self.library_populator.create_dataset_request(library, file_type="txt", contents="create_test") - create_response = self._post("libraries/%s/contents" % library["id"], payload, files=files) - self._assert_status_code_is(create_response, 200) - library_datasets = create_response.json() - assert len(library_datasets) == 1 - library_dataset = library_datasets[0] - - def show(): - return self._get("libraries/%s/contents/%s" % (library["id"], library_dataset["id"])) - - wait_on_state(show, assert_ok=True) - library_dataset = show().json() + library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True) self._assert_has_keys(library_dataset, "peek", "data_type") assert library_dataset["peek"].find("create_test") >= 0 assert library_dataset["file_ext"] == "txt", library_dataset["file_ext"] diff --git a/test/base/populators.py b/test/base/populators.py index 46a9557ccab..b9c3d4a33c7 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -446,6 +446,26 @@ class LibraryPopulator(object): wait_on_state(show, timeout=DEFAULT_TIMEOUT) return show().json() + def show_ldda(self, library_id, library_dataset_id): + return self.api_test_case.galaxy_interactor.get("libraries/%s/contents/%s" % (library_id, library_dataset_id)) + + def new_library_dataset_in_private_library(self, library_name="private_dataset", wait=True): + library = self.new_private_library(library_name) + payload, files = self.create_dataset_request(library, file_type="txt", contents="create_test") + create_response = self.api_test_case.galaxy_interactor.post("libraries/%s/contents" % library["id"], payload, files=files) + api_asserts.assert_status_code_is(create_response, 200) + library_datasets = create_response.json() + assert len(library_datasets) == 1 + library_dataset = library_datasets[0] + if wait: + def show(): + return self.show_ldda(library["id"], library_dataset["id"]) + + wait_on_state(show, assert_ok=True) + library_dataset = show().json() + + return library, library_dataset + class BaseDatasetCollectionPopulator(object): From 8a41921b49106276d1e8ed925464e734c07664d8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 14 Nov 2017 09:30:26 -0500 Subject: [PATCH 3/9] Respect permissions when creating HDAs/HDCAs from LDDAs. --- lib/galaxy/webapps/galaxy/api/history_contents.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 6af7b365101..e1fc4e5fc45 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -299,7 +299,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary # copy from library dataset hda = None if source == 'library': - hda = self.__create_hda_from_ldda(trans, content, history, check_ownership=False, check_accessible=False) + hda = self.__create_hda_from_ldda(trans, content, history) # copy an existing, accessible hda elif source == 'hda': @@ -315,9 +315,9 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary return self.hda_serializer.serialize_to_view(hda, user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed')) - def __create_hda_from_ldda(self, trans, content, history, check_ownership, check_accessible): + def __create_hda_from_ldda(self, trans, content, history, check_ownership=False, check_accessible=True): hda = None - ld = self.get_library_dataset(trans, content, check_ownership=False, check_accessible=False) + ld = self.get_library_dataset(trans, content, check_ownership=check_ownership, check_accessible=check_accessible) if type(ld) is not trans.app.model.LibraryDataset: raise exceptions.RequestParameterInvalidException( "Library content id ( %s ) is not a dataset" % content) @@ -410,7 +410,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary for ei in payload.get('element_identifiers'): # Convert lddas to hdas since there is no direct representation of library items in history. if ei['src'] == 'ldda': - hda = self.__create_hda_from_ldda(trans, ei['id'], history, check_ownership=False, check_accessible=False) + hda = self.__create_hda_from_ldda(trans, ei['id'], history) converted_identifiers.append({"name": ei["name"], "src": "hda", "id": trans.security.encode_id(hda.id)}) changed = True else: From 25e53e13453d594c56cf6f48474d74c7a9a09368 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 14 Nov 2017 09:30:55 -0500 Subject: [PATCH 4/9] API tests for creating HDCAs from LDDA elements. --- test/api/test_history_contents.py | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/test/api/test_history_contents.py b/test/api/test_history_contents.py index f395452f97c..ccb97fe4781 100644 --- a/test/api/test_history_contents.py +++ b/test/api/test_history_contents.py @@ -19,6 +19,7 @@ class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets): super(HistoryContentsApiTestCase, self).setUp() self.history_id = self._new_history() self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor) + self.library_populator = LibraryPopulator(self) def test_index_hda_summary(self): hda1 = self._new_dataset(self.history_id) @@ -57,7 +58,7 @@ class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets): assert self.__count_contents(second_history_id) == 1 def test_library_copy(self): - ld = LibraryPopulator(self).new_library_dataset("lda_test_library") + ld = self.library_populator.new_library_dataset("lda_test_library") create_data = dict( source='library', content=ld["id"], @@ -211,6 +212,43 @@ class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets): self.__check_create_collection_response(create_response) assert len(self._get("histories/%s/contents/dataset_collections" % second_history_id).json()) == 1 + def test_hdca_from_library_datasets(self): + ld = self.library_populator.new_library_dataset("el1") + ldda_id = ld["ldda_id"] + element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] + create_data = dict( + history_id=self.history_id, + type="dataset_collection", + name="Test From Library", + element_identifiers=json.dumps(element_identifiers), + collection_type="list", + ) + create_response = self._post("histories/%s/contents/dataset_collections" % self.history_id, create_data) + hdca = self.__check_create_collection_response(create_response) + elements = hdca["elements"] + assert len(elements) == 1 + hda = elements[0]["object"] + assert hda["hda_ldda"] == "hda" + assert hda["history_content_type"] == "dataset" + assert hda["copied_from_ldda_id"] == ldda_id + + def test_hdca_from_inaccessible_library_datasets(self): + library, library_dataset = self.library_populator.new_library_dataset_in_private_library("HDCACreateInaccesibleLibrary") + ldda_id = library_dataset["id"] + element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}] + create_data = dict( + history_id=self.history_id, + type="dataset_collection", + name="Test From Library", + element_identifiers=json.dumps(element_identifiers), + collection_type="list", + ) + with self._different_user(): + second_history_id = self._new_history() + create_response = self._post("histories/%s/contents/dataset_collections" % second_history_id, create_data) + # TODO: This should be 403 and a proper JSON response. + self._assert_status_code_is(create_response, 400) + def __check_create_collection_response(self, response): self._assert_status_code_is(response, 200) dataset_collection = response.json() From 44447ed08940782a4d022cd2dcddbdb2c2813ff4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 14 Nov 2017 15:25:51 -0500 Subject: [PATCH 5/9] Use github instead of NCBI for problematic data manager test FASTA. Seems to be failing many of the integration tool runs I've seen today. I've seen a couple different errors: ``` | File "/tmp/tmpVp6xM0/toolshed.g2.bx.psu.edu/repos/devteam/data_manager_fetch_genome_dbkeys_all_fasta/b1bc53e9bbc5/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py", line 312, in download_from_ncbi | return get_stream_reader(urlopen(url), tmp_dir) | File "/usr/lib/python2.7/urllib2.py", line 154, in urlopen | return opener.open(url, data, timeout) | File "/usr/lib/python2.7/urllib2.py", line 435, in open | response = meth(req, response) | File "/usr/lib/python2.7/urllib2.py", line 548, in http_response | 'http', request, response, code, msg, hdrs) | File "/usr/lib/python2.7/urllib2.py", line 473, in error | return self._call_chain(*args) | File "/usr/lib/python2.7/urllib2.py", line 407, in _call_chain | result = func(*args) | File "/usr/lib/python2.7/urllib2.py", line 556, in http_error_default | raise HTTPError(req.get_full_url(), code, msg, hdrs, fp) ``` and ``` Timed out after 60.25 seconds waiting on state. -------------------- >> begin captured stdout << --------------------- Problem in history with id adb5f5c93f827949 - summary of datasets below. -------------------------------------- | 1 - Create DBKey and Reference Genome (HID - NAME) | Dataset State: | queued | Dataset Blurb: | queued | Dataset Info: | None | Peek: | None | Dataset Job Standard Output: | None | Dataset Job Standard Error: | None | -------------------------------------- ``` --- test/integration/test_data_manager_table_reload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_data_manager_table_reload.py b/test/integration/test_data_manager_table_reload.py index e2c9cea8fe4..dae7e670ecc 100644 --- a/test/integration/test_data_manager_table_reload.py +++ b/test/integration/test_data_manager_table_reload.py @@ -37,8 +37,8 @@ FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT = {"dbkey_source|dbkey_source_selector": "ne "dbkey_source|dbkey_name": "NC_001617.1", "sequence_name": "NC_001617.1", "sequence_id": "NC_001617.1", - "reference_source|reference_source_selector": "ncbi", - "reference_source|requested_identifier": "NC_001617.1", + "reference_source|reference_source_selector": "url", + "reference_source|user_url": "https://raw.githubusercontent.com/galaxyproject/galaxy-test-data/master/NC_001617.1.fasta", "sorting|sort_selector": "as_is"} SAM_FASTA_ID = "toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_index_builder/sam_fasta_index_builder/0.0.2" SAM_FASTA_INPUT = {"all_fasta_source": "NC_001617.1", "sequence_name": "", "sequence_id": ""} From d5390350b8cfbdb4910cf0e671dbffb66f27f9e4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 15 Nov 2017 17:06:56 +0000 Subject: [PATCH 6/9] parse_interpreter: log a warning only if interpreter is set Also return None instead of False if there is no ``. Fix typo in warning. --- lib/galaxy/tools/parser/xml.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 266839a7151..08c6649ec76 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -127,12 +127,13 @@ class XmlToolSource(ToolSource): return environment_variables def parse_interpreter(self): + interpreter = None command_el = self._command_el - interpreter = (command_el is not None) and command_el.get("interpreter", None) - if not self.legacy_defaults: - log.warning("Deprecated interpeter attribute on command element is now ignored.") + if command_el is not None: + interpreter = command_el.get("interpreter", None) + if interpreter and not self.legacy_defaults: + log.warning("Deprecated interpreter attribute on command element is now ignored.") interpreter = None - return interpreter def parse_version_command(self): From abe1b240c33c0a2031a56fc52b8bcd5ec34ae27d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 15 Nov 2017 12:49:21 -0500 Subject: [PATCH 7/9] Fix broken library test I recently added. --- test/api/test_libraries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index 0029fb04bdc..8050157e054 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -87,7 +87,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): with self._different_user(): response = self.library_populator.show_ldda(library["id"], library_dataset["id"]) # TODO: this should really be 403 and a proper JSON exception. - self._assert_status_code_is(response.status_code, 400) + self._assert_status_code_is(response, 400) def test_create_dataset(self): library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True) From d3b5fac7516408c7d86e65b2899b7c6a8c9887e8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 15 Nov 2017 13:45:40 -0500 Subject: [PATCH 8/9] Test renaming multiple different outputs in workflows. Attempting to exercise a bug reported by @lparsons. --- test/api/test_workflows.py | 23 +++++++++++++++++++++++ test/functional/tools/create_2.xml | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index f3c45c03368..f1583fd1bd2 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -1355,6 +1355,29 @@ test_data: name = content["name"] assert name == "my new name", name + @skip_without_tool("create_2") + def test_run_rename_multiple_outputs(self): + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" +class: GalaxyWorkflow +inputs: [] +steps: + - tool_id: create_2 + state: + sleep_time: 0 + outputs: + out_file1: + rename: "my new name" + out_file2: + rename: "my other new name" +test_data: {} +""", history_id=history_id) + details1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=True) + details2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) + + assert details1["name"] == "my new name" + assert details2["name"] == "my other new name" + @skip_without_tool("cat") def test_run_rename_based_on_input(self): history_id = self.dataset_populator.new_history() diff --git a/test/functional/tools/create_2.xml b/test/functional/tools/create_2.xml index f9827846f64..f04bab7f4a9 100644 --- a/test/functional/tools/create_2.xml +++ b/test/functional/tools/create_2.xml @@ -1,5 +1,5 @@ - '$out_file1'; echo "2" > '$out_file2'; sleep '$sleep_time'; From 7157600f85993eb8570079d7c7d5b82d0e2af3f6 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 15 Nov 2017 16:39:26 -0500 Subject: [PATCH 9/9] fix few stray wiki links --- lib/galaxy/tools/xsd/galaxy.xsd | 14 +++++++------- templates/user/register.mako | 2 +- tools/data_source/microbial_import.xml | 8 -------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index f2bda039f76..506f0c0e812 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -471,8 +471,8 @@ This function is called before the tool is executed. If it raises any exceptions ```python def validate(incoming): """Validator for the plotting program""" - - + + bins = incoming.get("bins","") col = incoming.get("col","") @@ -523,8 +523,8 @@ This code executes after the background process running the tool finishes its ru from galaxy import datatypes def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr): ext = param_dict.get('extension', 'text') - items = out_data.items() - for name, data in items: + items = out_data.items() + for name, data in items: newdata = datatypes.factory(ext)(id=data.id) for key, value in data. __dict__.items(): setattr(newdata, key, value) @@ -2120,7 +2120,7 @@ effectively be a list with one ``None``-like entity in it. The following idiom can be used to iterate over such a list and build a hypothetical ``-B`` parameter for each file - the ``if`` block is used to handle the case where a ``None``-like -entity appears in the list because no files were selected: +entity appears in the list because no files were selected: ``` #for $input in $input1 @@ -2830,7 +2830,7 @@ exclusively use ``filter``s to populate options. ### ``from_data_table`` See Galaxy's -[data tables documentation](https://wiki.galaxyproject.org/Admin/Tools/Data%20Tables) +[data tables documentation](https://galaxyproject.org/admin/tools/data-tables) for information on setting up data tables. Once a data table has been configured and populated, these can be easily @@ -3132,7 +3132,7 @@ specified by the ``skip`` attribute. Tool data table name to check against if ``type`` is ``dataset_metadata_in_tool_data``. See the documentation for -[tool data tables](https://wiki.galaxyproject.org/Admin/Tools/Data%20Tables) +[tool data tables](https://galaxyproject.org/admin/tools/data-tables) and [data managers](https://galaxyproject.org/admin/tools/data-managers/) for more information. diff --git a/templates/user/register.mako b/templates/user/register.mako index 4bfcf993ab6..659e32a0ebd 100644 --- a/templates/user/register.mako +++ b/templates/user/register.mako @@ -173,7 +173,7 @@ def inherit(context): <% subscribe_check_box.checked = True %> %endif ${subscribe_check_box.get_html()} -

See +

See all Galaxy project mailing lists.

%endif diff --git a/tools/data_source/microbial_import.xml b/tools/data_source/microbial_import.xml index b07f557cb7e..5b817b62c8d 100644 --- a/tools/data_source/microbial_import.xml +++ b/tools/data_source/microbial_import.xml @@ -102,13 +102,5 @@ Current datasets available include ----- Organisms in **bold** are available at the UCSC Browser. - ------ - -.. class:: infomark - -**Note:** Having trouble locating your organism? Click here_ for a list of available species and their location. - -.. _here: https://wiki.galaxyproject.org/Main/Data%20Libraries/Microbes