Merge remote-tracking branch 'origin/dev' into job-handler-mules

lib/galaxy/tools/parser/xml.py

@@ -127,12 +127,13 @@ class XmlToolSource(ToolSource):
         return environment_variables
 
     def parse_interpreter(self):
+        interpreter = None
         command_el = self._command_el
-        interpreter = (command_el is not None) and command_el.get("interpreter", None)
-        if not self.legacy_defaults:
-            log.warning("Deprecated interpeter attribute on command element is now ignored.")
+        if command_el is not None:
+            interpreter = command_el.get("interpreter", None)
+        if interpreter and not self.legacy_defaults:
+            log.warning("Deprecated interpreter attribute on command element is now ignored.")
             interpreter = None
-
         return interpreter
 
     def parse_version_command(self):

lib/galaxy/tools/xsd/galaxy.xsd

@@ -471,8 +471,8 @@ This function is called before the tool is executed. If it raises any exceptions
 ```python
 def validate(incoming):
     """Validator for the plotting program"""
-    
-    
+
+
     bins = incoming.get("bins","")
     col = incoming.get("col","")
 
@@ -523,8 +523,8 @@ This code executes after the background process running the tool finishes its ru
 from galaxy import datatypes
 def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr):
     ext = param_dict.get('extension', 'text')
-    items = out_data.items()   
-    for name, data in items: 
+    items = out_data.items()
+    for name, data in items:
         newdata = datatypes.factory(ext)(id=data.id)
         for key, value in data. __dict__.items():
             setattr(newdata, key, value)
@@ -2120,7 +2120,7 @@ effectively be a list with one ``None``-like entity in it.
 
 The following idiom can be used to iterate over such a list and build a hypothetical ``-B``
 parameter for each file - the ``if`` block is used to handle the case where a ``None``-like
-entity appears in the list because no files were selected: 
+entity appears in the list because no files were selected:
 
 ```
 #for $input in $input1
@@ -2830,7 +2830,7 @@ exclusively use ``filter``s to populate options.
 ### ``from_data_table``
 
 See Galaxy's
-[data tables documentation](https://wiki.galaxyproject.org/Admin/Tools/Data%20Tables)
+[data tables documentation](https://galaxyproject.org/admin/tools/data-tables)
 for information on setting up data tables.
 
 Once a data table has been configured and populated, these can be easily
@@ -3132,7 +3132,7 @@ specified by the ``skip`` attribute.</xs:documentation>
           <xs:annotation>
             <xs:documentation xml:lang="en">Tool data table name to check against
 if ``type`` is ``dataset_metadata_in_tool_data``. See the documentation for
-[tool data tables](https://wiki.galaxyproject.org/Admin/Tools/Data%20Tables)
+[tool data tables](https://galaxyproject.org/admin/tools/data-tables)
 and [data managers](https://galaxyproject.org/admin/tools/data-managers/) for
 more information.</xs:documentation>
           </xs:annotation>

lib/galaxy/webapps/galaxy/api/history_contents.py

@@ -307,13 +307,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
         # copy from library dataset
         hda = None
         if source == 'library':
-            ld = self.get_library_dataset(trans, content)
-            # TODO: why would get_library_dataset NOT return a library dataset?
-            if type(ld) is not trans.app.model.LibraryDataset:
-                raise exceptions.RequestParameterInvalidException(
-                    "Library content id ( %s ) is not a dataset" % content)
-            # insert into history
-            hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True)
+            hda = self.__create_hda_from_ldda(trans, content, history)
 
         # copy an existing, accessible hda
         elif source == 'hda':
@@ -329,6 +323,15 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
         return self.hda_serializer.serialize_to_view(hda,
             user=trans.user, trans=trans, **self._parse_serialization_params(kwd, 'detailed'))
 
+    def __create_hda_from_ldda(self, trans, content, history):
+        hda = None
+        ld = self.get_library_dataset(trans, content)
+        if type(ld) is not trans.app.model.LibraryDataset:
+            raise exceptions.RequestParameterInvalidException(
+                "Library content id ( %s ) is not a dataset" % content)
+        hda = ld.library_dataset_dataset_association.to_history_dataset_association(history, add_to_history=True)
+        return hda
+
     def __create_datasets_from_library_folder(self, trans, history, payload, **kwd):
         rval = []
 
@@ -374,10 +377,54 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
         return rval
 
     def __create_dataset_collection(self, trans, history, payload, **kwd):
+        """Create hdca in a history from the list of element identifiers
+
+        :param history: history the new hdca should be added to
+        :type  history: History
+        :param source: whether to create a new collection or copy existing one
+        :type  source: str
+        :param payload: dictionary structure containing:
+            :param collection_type: type (and depth) of the new collection
+            :type name: str
+            :param element_identifiers: list of elements that should be in the new collection
+                :param element: one member of the collection
+                    :param name: name of the element
+                    :type name: str
+                    :param src: source of the element (hda/ldda)
+                    :type src: str
+                    :param id: identifier
+                    :type id: str
+                :type element: dict
+            :type name: list
+            :param name: name of the collection
+            :type name: str
+            :param hide_source_items: whether to mark the original hdas as hidden
+            :type name: bool
+        :type  payload: dict
+
+       .. note:: Elements may be nested depending on the collection_type
+
+        :returns:   dataset collection information
+        :rtype:     dict
+
+        :raises: RequestParameterInvalidException, RequestParameterMissingException
+        """
         source = kwd.get("source", payload.get("source", "new_collection"))
         service = trans.app.dataset_collections_service
         if source == "new_collection":
             create_params = api_payload_to_create_params(payload)
+            converted_identifiers = []
+            changed = False
+            for ei in payload.get('element_identifiers'):
+                # Convert lddas to hdas since there is no direct representation of library items in history.
+                if ei['src'] == 'ldda':
+                    hda = self.__create_hda_from_ldda(trans, ei['id'], history)
+                    converted_identifiers.append({"name": ei["name"], "src": "hda", "id": trans.security.encode_id(hda.id)})
+                    changed = True
+                else:
+                    converted_identifiers.append(ei)
+            if changed:
+                create_params['element_identifiers'] = converted_identifiers
             dataset_collection_instance = service.create(
                 trans,
                 parent=history,

templates/user/register.mako

@@ -173,7 +173,7 @@ def inherit(context):
                         <% subscribe_check_box.checked = True %>
                     %endif
                     ${subscribe_check_box.get_html()}
-                    <p>See <a href="http://galaxyproject.org/wiki/Mailing%20Lists" target="_blank">
+                    <p>See <a href="https://galaxyproject.org/mailing-lists/" target="_blank">
                     all Galaxy project mailing lists</a>.</p>
                 </div>
             %endif

test/api/test_history_contents.py

@@ -19,6 +19,7 @@ class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets):
         super(HistoryContentsApiTestCase, self).setUp()
         self.history_id = self._new_history()
         self.dataset_collection_populator = DatasetCollectionPopulator(self.galaxy_interactor)
+        self.library_populator = LibraryPopulator(self)
 
     def test_index_hda_summary(self):
         hda1 = self._new_dataset(self.history_id)
@@ -57,7 +58,7 @@ class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets):
         assert self.__count_contents(second_history_id) == 1
 
     def test_library_copy(self):
-        ld = LibraryPopulator(self).new_library_dataset("lda_test_library")
+        ld = self.library_populator.new_library_dataset("lda_test_library")
         create_data = dict(
             source='library',
             content=ld["id"],
@@ -222,6 +223,43 @@ class HistoryContentsApiTestCase(api.ApiTestCase, TestsDatasets):
         self.__check_create_collection_response(create_response)
         assert len(self._get("histories/%s/contents/dataset_collections" % second_history_id).json()) == 1
 
+    def test_hdca_from_library_datasets(self):
+        ld = self.library_populator.new_library_dataset("el1")
+        ldda_id = ld["ldda_id"]
+        element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}]
+        create_data = dict(
+            history_id=self.history_id,
+            type="dataset_collection",
+            name="Test From Library",
+            element_identifiers=json.dumps(element_identifiers),
+            collection_type="list",
+        )
+        create_response = self._post("histories/%s/contents/dataset_collections" % self.history_id, create_data)
+        hdca = self.__check_create_collection_response(create_response)
+        elements = hdca["elements"]
+        assert len(elements) == 1
+        hda = elements[0]["object"]
+        assert hda["hda_ldda"] == "hda"
+        assert hda["history_content_type"] == "dataset"
+        assert hda["copied_from_ldda_id"] == ldda_id
+
+    def test_hdca_from_inaccessible_library_datasets(self):
+        library, library_dataset = self.library_populator.new_library_dataset_in_private_library("HDCACreateInaccesibleLibrary")
+        ldda_id = library_dataset["id"]
+        element_identifiers = [{"name": "el1", "src": "ldda", "id": ldda_id}]
+        create_data = dict(
+            history_id=self.history_id,
+            type="dataset_collection",
+            name="Test From Library",
+            element_identifiers=json.dumps(element_identifiers),
+            collection_type="list",
+        )
+        with self._different_user():
+            second_history_id = self._new_history()
+            create_response = self._post("histories/%s/contents/dataset_collections" % second_history_id, create_data)
+            # TODO: This should be 403 and a proper JSON response.
+            self._assert_status_code_is(create_response, 400)
+
     def __check_create_collection_response(self, response):
         self._assert_status_code_is(response, 200)
         dataset_collection = response.json()

test/api/test_libraries.py

@@ -4,7 +4,6 @@ from base.populators import (
     DatasetPopulator,
     LibraryPopulator,
     TestsDatasets,
-    wait_on_state
 )
 
 
@@ -83,20 +82,15 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets):
             create_response = self._post("folders/%s/contents" % folder_id, payload)
             self._assert_status_code_is(create_response, 403)
 
+    def test_show_private_dataset_permissions(self):
+        library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
+        with self._different_user():
+            response = self.library_populator.show_ldda(library["id"], library_dataset["id"])
+            # TODO: this should really be 403 and a proper JSON exception.
+            self._assert_status_code_is(response, 400)
+
     def test_create_dataset(self):
-        library = self.library_populator.new_private_library("ForCreateDatasets")
-        payload, files = self.library_populator.create_dataset_request(library, file_type="txt", contents="create_test")
-        create_response = self._post("libraries/%s/contents" % library["id"], payload, files=files)
-        self._assert_status_code_is(create_response, 200)
-        library_datasets = create_response.json()
-        assert len(library_datasets) == 1
-        library_dataset = library_datasets[0]
-
-        def show():
-            return self._get("libraries/%s/contents/%s" % (library["id"], library_dataset["id"]))
-
-        wait_on_state(show, assert_ok=True)
-        library_dataset = show().json()
+        library, library_dataset = self.library_populator.new_library_dataset_in_private_library("ForCreateDatasets", wait=True)
         self._assert_has_keys(library_dataset, "peek", "data_type")
         assert library_dataset["peek"].find("create_test") >= 0
         assert library_dataset["file_ext"] == "txt", library_dataset["file_ext"]

test/api/test_workflows.py

@@ -1355,6 +1355,29 @@ test_data:
         name = content["name"]
         assert name == "my new name", name
 
+    @skip_without_tool("create_2")
+    def test_run_rename_multiple_outputs(self):
+        with self.dataset_populator.test_history() as history_id:
+            self._run_jobs("""
+class: GalaxyWorkflow
+inputs: []
+steps:
+  - tool_id: create_2
+    state:
+      sleep_time: 0
+    outputs:
+      out_file1:
+        rename: "my new name"
+      out_file2:
+        rename: "my other new name"
+test_data: {}
+""", history_id=history_id)
+        details1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=True)
+        details2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2)
+
+        assert details1["name"] == "my new name"
+        assert details2["name"] == "my other new name"
+
     @skip_without_tool("cat")
     def test_run_rename_based_on_input(self):
         history_id = self.dataset_populator.new_history()

test/base/populators.py

@@ -471,6 +471,26 @@ class LibraryPopulator(object):
         wait_on_state(show, timeout=DEFAULT_TIMEOUT)
         return show().json()
 
+    def show_ldda(self, library_id, library_dataset_id):
+        return self.api_test_case.galaxy_interactor.get("libraries/%s/contents/%s" % (library_id, library_dataset_id))
+
+    def new_library_dataset_in_private_library(self, library_name="private_dataset", wait=True):
+        library = self.new_private_library(library_name)
+        payload, files = self.create_dataset_request(library, file_type="txt", contents="create_test")
+        create_response = self.api_test_case.galaxy_interactor.post("libraries/%s/contents" % library["id"], payload, files=files)
+        api_asserts.assert_status_code_is(create_response, 200)
+        library_datasets = create_response.json()
+        assert len(library_datasets) == 1
+        library_dataset = library_datasets[0]
+        if wait:
+            def show():
+                return self.show_ldda(library["id"], library_dataset["id"])
+
+            wait_on_state(show, assert_ok=True)
+            library_dataset = show().json()
+
+        return library, library_dataset
+
 
 class BaseDatasetCollectionPopulator(object):
 

test/functional/tools/create_2.xml

@@ -1,5 +1,5 @@
 <tool id="create_2" name="create_2">
-    <command><![CDATA[
+    <command detect_errors="exit_code"><![CDATA[
         echo "1" > '$out_file1';
         echo "2" > '$out_file2';
         sleep '$sleep_time';

test/integration/test_data_manager_table_reload.py

@@ -37,8 +37,8 @@ FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT = {"dbkey_source|dbkey_source_selector": "ne
                                        "dbkey_source|dbkey_name": "NC_001617.1",
                                        "sequence_name": "NC_001617.1",
                                        "sequence_id": "NC_001617.1",
-                                       "reference_source|reference_source_selector": "ncbi",
-                                       "reference_source|requested_identifier": "NC_001617.1",
+                                       "reference_source|reference_source_selector": "url",
+                                       "reference_source|user_url": "https://raw.githubusercontent.com/galaxyproject/galaxy-test-data/master/NC_001617.1.fasta",
                                        "sorting|sort_selector": "as_is"}
 SAM_FASTA_ID = "toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_index_builder/sam_fasta_index_builder/0.0.2"
 SAM_FASTA_INPUT = {"all_fasta_source": "NC_001617.1", "sequence_name": "", "sequence_id": ""}

tools/data_source/microbial_import.xml

@@ -102,13 +102,5 @@ Current datasets available include
 -----
 
 Organisms in **bold** are available at the UCSC Browser.
-
------
-
-.. class:: infomark
-
-**Note:** Having trouble locating your organism?  Click here_ for a list of available species and their location.
-
-.. _here: https://wiki.galaxyproject.org/Main/Data%20Libraries/Microbes
   </help>
 </tool>