From cbf3a62ef821a9ae05f988a699963e968b7bfade Mon Sep 17 00:00:00 2001
From: Anthony Bretaudeau <anthony.bretaudeau@inria.fr>
Date: Wed, 24 Mar 2021 11:42:35 +0100
Subject: [PATCH 1/8] Don't block data table deletion when using refgenie

---
 lib/galaxy/tools/data/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py
index 8ffe3a8491a..a2f64922e83 100644
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -893,7 +893,11 @@ class RefgenieToolDataTable(TabularToolDataTable):
         return rval
 
     def _remove_entry(self, values):
-        raise NotImplementedError("Not supported")
+
+        log.warning("Deletion from refgenie-backed '%s' data table is not supported, only deleting from .loc files", self.name)
+
+        # Update every non-refgenie files
+        super._remove_entry(values)
 
 
 def expand_here_template(content, here=None):

From 1f7170ff66d8760c9f33db428757e55961291108 Mon Sep 17 00:00:00 2001
From: Assunta DeSanto <assuntad23@gmail.com>
Date: Wed, 24 Mar 2021 12:21:25 -0400
Subject: [PATCH 2/8] removed reference to remove-extensions

---
 client/src/components/Collections/PairCollectionCreator.vue | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/components/Collections/PairCollectionCreator.vue b/client/src/components/Collections/PairCollectionCreator.vue
index 10b6d81130f..e4f2a283854 100644
--- a/client/src/components/Collections/PairCollectionCreator.vue
+++ b/client/src/components/Collections/PairCollectionCreator.vue
@@ -81,7 +81,6 @@
                     :hideSourceItems="hideSourceItems"
                     @onUpdateHideSourceItems="onUpdateHideSourceItems"
                     @clicked-create="clickedCreate"
-                    @remove-extensions-toggle="removeExtensionsToggle"
                     :creationFn="creationFn"
                 >
                     <template v-slot:help-content>

From 7e136c83b3942083cab742b3c28274103e68d977 Mon Sep 17 00:00:00 2001
From: Anthony Bretaudeau <anthony.bretaudeau@inria.fr>
Date: Thu, 25 Mar 2021 14:01:12 +0100
Subject: [PATCH 3/8] fix

---
 lib/galaxy/tools/data/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py
index a2f64922e83..34cc1dae9c4 100644
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -894,10 +894,10 @@ class RefgenieToolDataTable(TabularToolDataTable):
 
     def _remove_entry(self, values):
 
-        log.warning("Deletion from refgenie-backed '%s' data table is not supported, only deleting from .loc files", self.name)
+        log.warning("Deletion from refgenie-backed '%s' data table is not supported, will only try to delete from .loc files", self.name)
 
         # Update every non-refgenie files
-        super._remove_entry(values)
+        super()._remove_entry(values)
 
 
 def expand_here_template(content, here=None):

From 340caa1174278a26bb5158b9f916e5e5b75a94a7 Mon Sep 17 00:00:00 2001
From: Anthony Bretaudeau <anthony.bretaudeau@inria.fr>
Date: Thu, 25 Mar 2021 14:01:17 +0100
Subject: [PATCH 4/8] add more tests

---
 test/integration/refgenie.yml                 |   5 +
 test/integration/test_data_manager.py         |  56 ++++++++-
 .../integration/test_data_manager_refgenie.py | 115 ++++++++++++++++++
 3 files changed, 172 insertions(+), 4 deletions(-)
 create mode 100644 test/integration/refgenie.yml
 create mode 100644 test/integration/test_data_manager_refgenie.py

diff --git a/test/integration/refgenie.yml b/test/integration/refgenie.yml
new file mode 100644
index 00000000000..c1c5ac61c15
--- /dev/null
+++ b/test/integration/refgenie.yml
@@ -0,0 +1,5 @@
+config_version: 0.4
+genome_folder: /tmp/refgenie
+genome_servers:
+ - http://refgenomes.databio.org
+genomes: null
diff --git a/test/integration/test_data_manager.py b/test/integration/test_data_manager.py
index 61e706c2caa..e1eb581dfa0 100644
--- a/test/integration/test_data_manager.py
+++ b/test/integration/test_data_manager.py
@@ -79,14 +79,62 @@ class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesS
 
     def test_data_manager_manual(self):
         """
-        Test that data_manager_manual works, which uses a signigicant amount of Galaxy-internal code
+        Test that data_manager_manual works, which uses a significant amount of Galaxy-internal code
         """
         self.install_repository('iuc', 'data_manager_manual', '1ed87dee9e68')
         with self._different_user(email="%s@galaxy.org" % self.username):
             with self.dataset_populator.test_history() as history_id:
-                self.dataset_populator.run_tool(tool_id=DATA_MANAGER_MANUAL_ID,
-                                                inputs=DATA_MANAGER_MANUAL_INPUT,
-                                                history_id=history_id)
+                run_response = self.dataset_populator.run_tool(tool_id=DATA_MANAGER_MANUAL_ID,
+                                                               inputs=DATA_MANAGER_MANUAL_INPUT,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
+
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'dm6', 'dbkey')
+        assert 'dm6' in entries
+
+        table_content = {line[0]: line for line in self._app.tool_data_tables.get("all_fasta").to_dict(view="element")['fields']}
+
+        self._app.tool_data_tables.get("all_fasta").remove_entry(table_content['dm6'])
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'dm6', 'dbkey')
+        assert entries is None
+
+    def test_data_manager_manual_multiple(self):
+        """
+        Test adding/removing on the same data table with multiple data managers
+        """
+        self.install_repository("devteam", "data_manager_fetch_genome_dbkeys_all_fasta", "14eb0fc65c62")
+        self.install_repository('iuc', 'data_manager_manual', '1ed87dee9e68')
+        with self._different_user(email="%s@galaxy.org" % self.username):
+            with self.dataset_populator.test_history() as history_id:
+                run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,
+                                                               inputs=FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
+                run_response = self.dataset_populator.run_tool(tool_id=DATA_MANAGER_MANUAL_ID,
+                                                               inputs=DATA_MANAGER_MANUAL_INPUT,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
+
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'NC_001617.1', 'dbkey')
+        assert 'NC_001617.1' in entries
+
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'dm6', 'dbkey')
+        assert 'dm6' in entries
+
+        table_content = {line[0]: line for line in self._app.tool_data_tables.get("all_fasta").to_dict(view="element")['fields']}
+
+        self._app.tool_data_tables.get("all_fasta").remove_entry(table_content['dm6'])
+
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'dm6', 'dbkey')
+        assert entries is None
+
+        self._app.tool_data_tables.get("all_fasta").remove_entry(table_content['NC_001617.1'])
+
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'NC_001617.1', 'dbkey')
+        assert entries is None
 
     @classmethod
     def get_secure_ascii_digits(cls, n=12):
diff --git a/test/integration/test_data_manager_refgenie.py b/test/integration/test_data_manager_refgenie.py
new file mode 100644
index 00000000000..13ef28ba566
--- /dev/null
+++ b/test/integration/test_data_manager_refgenie.py
@@ -0,0 +1,115 @@
+import os
+import random
+import string
+
+from nose.plugins.skip import SkipTest
+
+from galaxy_test.base.populators import DatasetPopulator, skip_if_toolshed_down
+from galaxy_test.driver import integration_util
+from .uses_shed import CONDA_AUTO_INSTALL_JOB_TIMEOUT, UsesShed
+
+FETCH_TOOL_ID = 'toolshed.g2.bx.psu.edu/repos/devteam/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_fetch_genome_all_fasta_dbkey/0.0.3'
+FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT = {
+    "dbkey_source|dbkey_source_selector": "new",
+    "dbkey_source|dbkey": "NC_001617.1",
+    "dbkey_source|dbkey_name": "NC_001617.1",
+    "sequence_name": "NC_001617.1",
+    "sequence_id": "NC_001617.1",
+    "reference_source|reference_source_selector": "url",
+    "reference_source|user_url": "https://raw.githubusercontent.com/galaxyproject/galaxy-test-data/master/NC_001617.1.fasta",
+    "sorting|sort_selector": "as_is"
+}
+SAM_FASTA_ID = "toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_index_builder/sam_fasta_index_builder/0.0.3"
+SAM_FASTA_INPUT = {"all_fasta_source": "NC_001617.1", "sequence_name": "", "sequence_id": ""}
+DATA_MANAGER_MANUAL_ID = 'toolshed.g2.bx.psu.edu/repos/iuc/data_manager_manual/data_manager_manual/0.0.2'
+DATA_MANAGER_MANUAL_INPUT = {
+    "data_tables_0|data_table_name": "all_fasta",
+    "data_tables_0|columns_0|data_table_column_name": "value",
+    "data_tables_0|columns_0|data_table_column_value": "dm6",
+    "data_tables_0|columns_1|data_table_column_name": "name",
+    "data_tables_0|columns_1|data_table_column_value": "dm6",
+    "data_tables_0|columns_2|data_table_column_name": "dbkey",
+    "data_tables_0|columns_2|data_table_column_value": "dm6",
+    "data_tables_0|columns_3|data_table_column_name": "path",
+    "data_tables_0|columns_3|data_table_column_value": "dm6.fa",
+}
+DATA_MANAGER_MANUAL_INPUT_DBKEY = {
+    "data_tables_0|data_table_name": "__dbkeys__",
+    "data_tables_0|columns_0|data_table_column_name": "value",
+    "data_tables_0|columns_0|data_table_column_value": "dm7",
+    "data_tables_0|columns_1|data_table_column_name": "name",
+    "data_tables_0|columns_1|data_table_column_value": "dm7",
+    "data_tables_0|columns_2|data_table_column_name": "len_path",
+    "data_tables_0|columns_2|data_table_column_value": "dm7.len",
+}
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+REFGENIE_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "refgenie.yml")
+
+
+class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed):
+
+    """Test data manager installation and table reload through the API"""
+
+    framework_tool_and_types = True
+    use_shared_connection_for_amqp = True
+
+    def setUp(self):
+        super().setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        try:
+            import watchdog  # noqa: F401
+        except ImportError:
+            raise SkipTest("watchdog library is not available")
+        cls.configure_shed_and_conda(config)
+        config["tool_data_path"] = cls.shed_tool_data_dir
+        config["watch_tool_data_dir"] = True
+        cls.username = cls.get_secure_ascii_digits()
+        config["admin_users"] = "%s@galaxy.org" % cls.username
+        config["refgenie_config_file"] = REFGENIE_CONFIG_FILE
+
+    def test_data_manager_manual_refgenie(self):
+        """
+        Test that data_manager_manual works with refgenie enabled, which uses a significant amount of Galaxy-internal code
+        """
+        self.install_repository('iuc', 'data_manager_manual', '1ed87dee9e68')
+        with self._different_user(email="%s@galaxy.org" % self.username):
+            with self.dataset_populator.test_history() as history_id:
+                run_response = self.dataset_populator.run_tool(tool_id=DATA_MANAGER_MANUAL_ID,
+                                                               inputs=DATA_MANAGER_MANUAL_INPUT,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
+
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'dm6', 'dbkey')
+        assert 'dm6' in entries
+
+        self._app.tool_data_tables.get("all_fasta").remove_entry(self._app.tool_data_tables.get("all_fasta").to_dict(view="element")['fields'][0])
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('dbkey', 'dm6', 'dbkey')
+        assert entries is None
+
+    def test_data_manager_manual_refgenie_dbkeys(self):
+        """
+        Test that data_manager_manual works with refgenie enabled, with a table defined first by refgenie
+        """
+        self.install_repository('iuc', 'data_manager_manual', '1ed87dee9e68')
+        with self._different_user(email="%s@galaxy.org" % self.username):
+            with self.dataset_populator.test_history() as history_id:
+                run_response = self.dataset_populator.run_tool(tool_id=DATA_MANAGER_MANUAL_ID,
+                                                               inputs=DATA_MANAGER_MANUAL_INPUT_DBKEY,
+                                                               history_id=history_id,
+                                                               assert_ok=False)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
+
+        entries = self._app.tool_data_tables.get("__dbkeys__").get_entries('name', 'dm7', 'name')
+        assert 'dm7' in entries
+
+        self._app.tool_data_tables.get("__dbkeys__").remove_entry(self._app.tool_data_tables.get("__dbkeys__").to_dict(view="element")['fields'][0])
+        entries = self._app.tool_data_tables.get("all_fasta").get_entries('name', 'dm7', 'name')
+        assert entries is None
+
+    @classmethod
+    def get_secure_ascii_digits(cls, n=12):
+        return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12))

From b5ad804b64cc3dff7c6a43f392057cc737dda627 Mon Sep 17 00:00:00 2001
From: Anthony Bretaudeau <anthony.bretaudeau@inria.fr>
Date: Thu, 25 Mar 2021 14:08:14 +0100
Subject: [PATCH 5/8] lint

---
 test/integration/test_data_manager_refgenie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/test_data_manager_refgenie.py b/test/integration/test_data_manager_refgenie.py
index 13ef28ba566..73e4d63f023 100644
--- a/test/integration/test_data_manager_refgenie.py
+++ b/test/integration/test_data_manager_refgenie.py
@@ -4,7 +4,7 @@ import string
 
 from nose.plugins.skip import SkipTest
 
-from galaxy_test.base.populators import DatasetPopulator, skip_if_toolshed_down
+from galaxy_test.base.populators import DatasetPopulator
 from galaxy_test.driver import integration_util
 from .uses_shed import CONDA_AUTO_INSTALL_JOB_TIMEOUT, UsesShed
 

From d983fc72e884accf448c98fbb559af30b24e7c4e Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 26 Mar 2021 15:34:24 -0400
Subject: [PATCH 6/8] Exhibit bug when unzipping collections with nametags.

---
 lib/galaxy_test/api/test_tools.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index c834078fa21..faa693d9f5e 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -402,11 +402,21 @@ class ToolsTestCase(ApiTestCase, TestsTools):
 
     def test_unzip_nested(self):
         with self.dataset_populator.test_history() as history_id:
-            hdca_list_id = self.__build_nested_list(history_id)
+            response = self.dataset_collection_populator.upload_collection(history_id, "list:paired", elements=[
+                {
+                    "name": "test0",
+                    "elements": [
+                        {"src": "pasted", "paste_content": "123\n", "name": "forward", "ext": "txt", "tags": ["#foo"]},
+                        {"src": "pasted", "paste_content": "456\n", "name": "reverse", "ext": "txt", "tags": ["#bar"]},
+                    ]
+                }
+            ])
+            self._assert_status_code_is(response, 200)
+            hdca_id = response.json()["outputs"][0]["id"]
             inputs = {
                 "input": {
                     'batch': True,
-                    'values': [{'src': 'hdca', 'map_over_type': 'paired', 'id': hdca_list_id}],
+                    'values': [{'src': 'hdca', 'map_over_type': 'paired', 'id': hdca_id}],
                 }
             }
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)

From f360f6498fbdff6b04e4b01b85c6e88510f937cb Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Fri, 26 Mar 2021 16:48:15 -0400
Subject: [PATCH 7/8] Fix copy tags bug - broken in #10761 I think.

---
 lib/galaxy/model/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 2f570e377ec..65314db1c7d 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -3314,6 +3314,8 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
 
     def copy_tags_to(self, copy_tags=None):
         if copy_tags is not None:
+            if isinstance(copy_tags, dict):
+                copy_tags = copy_tags.values()
             for tag in copy_tags:
                 copied_tag = tag.copy(cls=HistoryDatasetAssociationTagAssociation)
                 self.tags.append(copied_tag)

From ca6f2fa68e6d3acbcfe38dfdc8e0583113d94f78 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 30 Mar 2021 12:36:59 +0200
Subject: [PATCH 8/8] Create install database with create_db.sh

---
 lib/galaxy/model/orm/scripts.py | 3 ++-
 scripts/create_db.py            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py
index e16fb608f82..95f38a2615e 100644
--- a/lib/galaxy/model/orm/scripts.py
+++ b/lib/galaxy/model/orm/scripts.py
@@ -136,8 +136,9 @@ def get_config(argv, use_argparse=True, cwd=None):
         db_url = properties["%sdatabase_connection" % config_prefix]
     else:
         db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % os.path.join(get_data_dir(properties), default_sqlite_file)
+    install_database_connection = properties.get('install_database_connection')
 
-    return dict(db_url=db_url, repo=repo, config_file=config_file, database=database)
+    return dict(db_url=db_url, repo=repo, config_file=config_file, database=database, install_database_connection=install_database_connection)
 
 
 def manage_db():
diff --git a/scripts/create_db.py b/scripts/create_db.py
index bd0c7aac83b..39d808c8065 100755
--- a/scripts/create_db.py
+++ b/scripts/create_db.py
@@ -33,7 +33,7 @@ log = logging.getLogger(__name__)
 def invoke_create():
     config = get_config(sys.argv)
     if config['database'] == 'galaxy':
-        create_db(config['db_url'], config['config_file'])
+        create_db(config['db_url'], config['config_file'], map_install_models=not config['install_database_connection'])
     elif config['database'] == 'tool_shed':
         create_tool_shed_db(config['db_url'])
     elif config['database'] == 'install':