From f7820e82380f7ea69ba56037a3f463352ef36807 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 27 Nov 2017 09:54:36 +0100
Subject: [PATCH 01/23] Do not write shed_tool_data_table_config in place

This should prevent errors such as:
```
galaxy.util ERROR 2017-11-17 09:20:12,842 Error parsing file /data/users/mvandenb/gx/config/shed_tool_data_table_conf.xml
Traceback (most recent call last):
  File "lib/galaxy/util/__init__.py", line 217, in parse_xml
    root = tree.parse(fname, parser=ElementTree.XMLParser(target=DoctypeSafeCallbackTarget()))
  File "/usr/lib/python2.7/xml/etree/ElementTree.py", line 657, in parse
    self._root = parser.close()
  File "/usr/lib/python2.7/xml/etree/ElementTree.py", line 1654, in close
    self._raiseerror(v)
  File "/usr/lib/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror
    raise err
ParseError: no element found: line 1, column 0
```
reported in https://github.com/galaxyproject/galaxy/issues/5031.
---
 lib/galaxy/tools/data/__init__.py         |  3 +-
 lib/galaxy/util/renamed_temporary_file.py | 42 +++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 lib/galaxy/util/renamed_temporary_file.py

diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py
index d3ec66072b7..95ea5c0470d 100644
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -21,6 +21,7 @@ import requests
 from galaxy import util
 from galaxy.util.dictifiable import Dictifiable
 from galaxy.util.odict import odict
+from galaxy.util.renamed_temporary_file import RenamedTemporaryFile
 
 log = logging.getLogger(__name__)
 
@@ -186,7 +187,7 @@ class ToolDataTableManager(object):
         # add new elems
         out_elems.extend(new_elems)
         out_path_is_new = not os.path.exists(full_path)
-        with open(full_path, 'wb') as out:
+        with RenamedTemporaryFile(full_path) as out:
             out.write('<?xml version="1.0"?>\n<tables>\n')
             for elem in out_elems:
                 out.write(util.xml_to_string(elem, pretty=True))
diff --git a/lib/galaxy/util/renamed_temporary_file.py b/lib/galaxy/util/renamed_temporary_file.py
new file mode 100644
index 00000000000..ecbfb7a38c3
--- /dev/null
+++ b/lib/galaxy/util/renamed_temporary_file.py
@@ -0,0 +1,42 @@
+"""Safely write file to temporary file and then move file into place."""
+# Copied from https://stackoverflow.com/a/12007885.
+import os
+import tempfile
+
+
+class RenamedTemporaryFile(object):
+    """
+    A temporary file object which will be renamed to the specified
+    path on exit.
+    """
+    def __init__(self, final_path, **kwargs):
+        tmpfile_dir = kwargs.pop('dir', None)
+
+        # Put temporary file in the same directory as the location for the
+        # final file so that an atomic move into place can occur.
+
+        if tmpfile_dir is None:
+            tmpfile_dir = os.path.dirname(final_path)
+
+        self.tmpfile = tempfile.NamedTemporaryFile(dir=tmpfile_dir, **kwargs)
+        self.final_path = final_path
+
+    def __getattr__(self, attr):
+        """
+        Delegate attribute access to the underlying temporary file object.
+        """
+        return getattr(self.tmpfile, attr)
+
+    def __enter__(self):
+        self.tmpfile.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is None:
+            self.tmpfile.delete = False
+            result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb)
+            os.rename(self.tmpfile.name, self.final_path)
+        else:
+            result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb)
+
+        return result

From a2b8e445d85b50265da52c06f017c9c78be953b6 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 27 Nov 2017 11:25:37 +0100
Subject: [PATCH 02/23] Fix installation of data manager revision updates

Without this updates of data managers fails with
```
AttributeError: 'NoneType' object has no attribute 'get'
  File "galaxy/web/framework/middleware/sentry.py", line 40, in __call__
    iterable = self.application(environ, start_response)
  File "/bioinfo/guests/mvandenb/galaxy/.venv/local/lib/python2.7/site-packages/paste/recursive.py", line 85, in __call__
    return self.application(environ, start_response)
  File "/bioinfo/guests/mvandenb/galaxy/.venv/local/lib/python2.7/site-packages/paste/httpexceptions.py", line 640, in __call__
    return self.application(environ, start_response)
  File "galaxy/web/framework/base.py", line 136, in __call__
    return self.handle_request(environ, start_response)
  File "galaxy/web/framework/base.py", line 215, in handle_request
    body = method(trans, **kwargs)
  File "galaxy/web/framework/decorators.py", line 98, in decorator
    return func(self, trans, *args, **kwargs)
  File "galaxy/webapps/galaxy/controllers/admin_toolshed.py", line 1849, in update_to_changeset_revision
    persist=True)
  File "tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py", line 30, in __init__
    metadata_dict=metadata_dict, user=None)
  File "tool_shed/metadata/metadata_generator.py", line 63, in __init__
    self.metadata_dict = {'shed_config_filename': self.shed_config_dict.get('config_filename', None)}
```
---
 lib/galaxy/model/tool_shed_install/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py
index 454213fa18b..5df66d569c0 100644
--- a/lib/galaxy/model/tool_shed_install/__init__.py
+++ b/lib/galaxy/model/tool_shed_install/__init__.py
@@ -143,7 +143,7 @@ class ToolShedRepository(object):
                             if tool_id in tool_ids:
                                 self.shed_config_filename = name
                                 return shed_tool_conf_dict
-        if self.includes_datatypes:
+        if self.includes_datatypes or self.includes_data_managers:
             # We need to search by file paths here, which is less desirable.
             tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(self.tool_shed)
             for shed_tool_conf_dict in app.toolbox.dynamic_confs(include_migrated_tool_conf=True):

From d44a20cef098abf905f8525c42eb7f37d39f4ad9 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 28 Nov 2017 17:50:22 +0100
Subject: [PATCH 03/23] Use a temporary ToolDataTableManager when generating
 metadata

Otherwise loaded data tables bleed into the app-wide data tables. This can be
seen with lots of data tables pointing to (non-existing, at that point)
temporary files after a metadata reset in the galaxy admin panel,
but it can also cause failure to install tools with entirely new data tables.
---
 lib/tool_shed/galaxy_install/install_manager.py |  2 +-
 lib/tool_shed/metadata/metadata_generator.py    | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 3f59fde5696..5cdd254607a 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -532,7 +532,7 @@ class InstallRepositoryManager(object):
                 tdtm.install_tool_data_tables(tool_shed_repository, tool_index_sample_files)
             if tool_data_table_elems:
                 self.app.tool_data_tables.add_new_entries_from_config_file(tool_data_table_conf_filename,
-                                                                           None,
+                                                                           self.app.config.shed_tool_data_path,
                                                                            self.app.config.shed_tool_data_table_config,
                                                                            persist=True)
         if 'tools' in irmm_metadata_dict:
diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py
index f6df6e9c7be..883ee58b1e8 100644
--- a/lib/tool_shed/metadata/metadata_generator.py
+++ b/lib/tool_shed/metadata/metadata_generator.py
@@ -7,6 +7,7 @@ from sqlalchemy import and_
 
 from galaxy import util
 from galaxy.tools.data_manager.manager import DataManager
+from galaxy.tools.data import ToolDataTableManager
 from galaxy.tools.loader_directory import looks_like_a_tool
 from galaxy.tools.parser.interface import TestCollectionDef
 from galaxy.web import url_for
@@ -362,11 +363,14 @@ class MetadataGenerator(object):
             # its table elements into memory.
             relative_path, filename = os.path.split(sample_file)
             if filename == 'tool_data_table_conf.xml.sample':
+                # We create a new ToolDataTableManager to avoid adding entries to the app-wide
+                # tool data tables. This is only used for checking that the data table is valid.
+                tool_data_tables = ToolDataTableManager(work_dir)
                 new_table_elems, error_message = \
-                    self.app.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
-                                                                               tool_data_path=work_dir,
-                                                                               shed_tool_data_table_config=work_dir,
-                                                                               persist=False)
+                    tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
+                                                                      tool_data_path=work_dir,
+                                                                      shed_tool_data_table_config=work_dir,
+                                                                      persist=False)
                 if error_message:
                     self.invalid_file_tups.append((filename, error_message))
         for root, dirs, files in os.walk(files_dir):

From 561f71ec50ee6708913d57955456faa1932c4cac Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 29 Nov 2017 18:33:19 +0100
Subject: [PATCH 04/23] Avoid manipulating global toll data tables

---
 lib/tool_shed/tools/data_table_manager.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py
index 9a89c1cb343..17d1f3ebc3c 100644
--- a/lib/tool_shed/tools/data_table_manager.py
+++ b/lib/tool_shed/tools/data_table_manager.py
@@ -3,6 +3,7 @@ import os
 import shutil
 from xml.etree import ElementTree as XmlET
 
+from galaxy.tools.data import ToolDataTableManager as tdtm
 from tool_shed.util import hg_util, xml_util
 
 log = logging.getLogger(__name__)
@@ -11,6 +12,7 @@ log = logging.getLogger(__name__)
 class ToolDataTableManager(object):
 
     def __init__(self, app):
+        self.tdtm = tdtm(app.config.shed_tool_data_path)
         self.app = app
 
     def generate_repository_info_elem(self, tool_shed, repository_name, changeset_revision, owner,
@@ -99,7 +101,7 @@ class ToolDataTableManager(object):
         error = False
         message = ''
         try:
-            new_table_elems, message = self.app.tool_data_tables \
+            new_table_elems, message = self.tdtm \
                 .add_new_entries_from_config_file(config_filename=filename,
                                                   tool_data_path=self.app.config.shed_tool_data_path,
                                                   shed_tool_data_table_config=self.app.config.shed_tool_data_table_config,
@@ -167,9 +169,9 @@ class ToolDataTableManager(object):
             # Remove old data_table
             os.unlink(tool_data_table_conf_filename)
             # Persist new data_table content.
-            self.app.tool_data_tables.to_xml_file(tool_data_table_conf_filename, elems)
+            self.tdtm.to_xml_file(tool_data_table_conf_filename, elems)
         return tool_data_table_conf_filename, elems
 
     def reset_tool_data_tables(self):
         # Reset the tool_data_tables to an empty dictionary.
-        self.app.tool_data_tables.data_tables = {}
+        self.tdtm.data_tables = {}

From c02d9363c45b7272f8b507f6c46aecf867faee5d Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 26 Nov 2017 18:59:28 +0100
Subject: [PATCH 05/23] Fix copying of loc.sample files

`filename` should be joined with the tool_path, as evidenced by other uses of
copy_sample_file.
---
 lib/tool_shed/util/tool_util.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py
index da0518b1659..958a299bd73 100644
--- a/lib/tool_shed/util/tool_util.py
+++ b/lib/tool_shed/util/tool_util.py
@@ -41,13 +41,13 @@ def build_tool_panel_section_select_field(app):
 
 def copy_sample_file(app, filename, dest_path=None):
     """
-    Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx.  The default value for dest_path
-    is ~/tool-data.
+    Copies a sample file at `filename` to `the dest_path`
+    directory and strips the '.sample' extensions from `filename`.
     """
     if dest_path is None:
         dest_path = os.path.abspath(app.config.tool_data_path)
     sample_file_name = basic_util.strip_path(filename)
-    copied_file = sample_file_name.replace('.sample', '')
+    copied_file = sample_file_name.rsplit('.sample', 1)[0]
     full_source_path = os.path.abspath(filename)
     full_destination_path = os.path.join(dest_path, sample_file_name)
     # Don't copy a file to itself - not sure how this happens, but sometimes it does...
@@ -159,7 +159,7 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup
                 for sample_file in sample_files:
                     sample_file_name = basic_util.strip_path(sample_file)
                     if sample_file_name == '%s.sample' % missing_file_name:
-                        copy_sample_file(app, sample_file)
+                        copy_sample_file(app, os.path.join(tool_path, sample_file))
                         if options.tool_data_table and options.tool_data_table.missing_index_file:
                             options.tool_data_table.handle_found_index_file(options.missing_index_file)
                         sample_files_copied.append(options.missing_index_file)

From fe6033fb22a23d4004a70de101e3f9597d0c6912 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 1 Dec 2017 09:44:18 +0100
Subject: [PATCH 06/23] Avoid global changes to app.config.tool_data_path

---
 lib/galaxy/webapps/tool_shed/controllers/repository.py | 3 ---
 lib/tool_shed/tools/tool_validator.py                  | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py
index e2ae53f3525..5a710847d8b 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -2884,7 +2884,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         tool_lineage = []
         tool = None
         guid = None
-        original_tool_data_path = trans.app.config.tool_data_path
         revision_label = hg_util.get_revision_label(trans.app, repository, changeset_revision, include_date=False)
         repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(trans.app, repository_id, changeset_revision)
         if repository_metadata:
@@ -2905,7 +2904,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
                                                                                  full_path_to_tool_config,
                                                                                  changeset_revision)
                             if can_use_disk_file:
-                                trans.app.config.tool_data_path = work_dir
                                 tool, valid, message, sample_files = \
                                     tv.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
                                                                                    repository_id,
@@ -2937,7 +2935,6 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
                                                                                            selected_value=changeset_revision,
                                                                                            add_id_to_name=False,
                                                                                            downloadable=False)
-        trans.app.config.tool_data_path = original_tool_data_path
         return trans.fill_template("/webapps/tool_shed/repository/view_tool_metadata.mako",
                                    render_repository_actions_for=render_repository_actions_for,
                                    repository=repository,
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index 1effbd784d7..4ab69d7eafe 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -219,7 +219,7 @@ class ToolValidator(object):
         # intended to help handle sample files that are in the manifest, but have been deleted from disk.
         sample_files, deleted_sample_files = self.get_list_of_copied_sample_files(repo, ctx, dir=work_dir)
         if sample_files:
-            self.app.config.tool_data_path = work_dir
+            # self.app.config.tool_data_path = work_dir
             if 'tool_data_table_conf.xml.sample' in sample_files:
                 # Load entries into the tool_data_tables if the tool requires them.
                 tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
@@ -256,7 +256,7 @@ class ToolValidator(object):
                                                                tool_config_filepath,
                                                                changeset_revision)
         if can_use_disk_file:
-            self.app.config.tool_data_path = work_dir
+            # self.app.config.tool_data_path = work_dir
             tool, valid, message, sample_files = \
                 self.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
                                                                  repository_id,

From 7df9dce63d635d66a0a9ce5045135ab49b7ef506 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 1 Dec 2017 09:48:31 +0100
Subject: [PATCH 07/23] Rename the ToolDataTableManager that deals with
 repository data to ShedToolDataTableManager to avoid confusion

---
 lib/galaxy/webapps/tool_shed/controllers/upload.py | 10 +++++-----
 lib/tool_shed/galaxy_install/install_manager.py    | 12 ++++++------
 lib/tool_shed/tools/data_table_manager.py          |  2 +-
 lib/tool_shed/tools/tool_validator.py              | 14 ++++++++------
 lib/tool_shed/util/commit_util.py                  |  6 +++---
 5 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py
index 8e16be8931c..7395035c0f9 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/upload.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py
@@ -17,7 +17,6 @@ from tool_shed.dependencies import attribute_handlers
 from tool_shed.galaxy_install import dependency_display
 from tool_shed.metadata import repository_metadata_manager
 from tool_shed.repository_types import util as rt_util
-from tool_shed.tools import data_table_manager
 from tool_shed.util import (
     basic_util,
     commit_util,
@@ -28,6 +27,7 @@ from tool_shed.util import (
     xml_util
 )
 from tool_shed.util.web_util import escape
+from tool_shed.tools.data_table_manager import ShedToolDataTableManager
 
 log = logging.getLogger(__name__)
 
@@ -99,7 +99,7 @@ class UploadController(BaseUIController):
             if uploaded_file or uploaded_directory:
                 rdah = attribute_handlers.RepositoryDependencyAttributeHandler(trans.app, unpopulate=False)
                 tdah = attribute_handlers.ToolDependencyAttributeHandler(trans.app, unpopulate=False)
-                tdtm = data_table_manager.ToolDataTableManager(trans.app)
+                stdtm = ShedToolDataTableManager(trans.app)
                 ok = True
                 isgzip = False
                 isbz2 = False
@@ -223,7 +223,7 @@ class UploadController(BaseUIController):
                                 # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
                                 # by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
                                 # dictionary.
-                                error, error_message = tdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False)
+                                error, error_message = stdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False)
                                 if error:
                                     message = '%s<br/>%s' % (message, error_message)
                             # See if the content of the change set was valid.
@@ -312,7 +312,7 @@ class UploadController(BaseUIController):
                         message += invalid_repository_dependencies_message
                         status = 'error'
                     # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
-                    tdtm.reset_tool_data_tables()
+                    stdtm.reset_tool_data_tables()
                     if uploaded_directory:
                         basic_util.remove_dir(uploaded_directory)
                     trans.response.send_redirect(web.url_for(controller='repository',
@@ -326,7 +326,7 @@ class UploadController(BaseUIController):
                         basic_util.remove_dir(uploaded_directory)
                     status = 'error'
                 # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
-                tdtm.reset_tool_data_tables()
+                stdtm.reset_tool_data_tables()
         return trans.fill_template('/webapps/tool_shed/repository/upload.mako',
                                    repository=repository,
                                    changeset_revision=tip,
diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 5cdd254607a..490c172b903 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -19,7 +19,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment impor
 from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager
 from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager
 from tool_shed.galaxy_install.tools import data_manager, tool_panel_manager
-from tool_shed.tools import data_table_manager
+from tool_shed.tools.data_table_manager import ShedToolDataTableManager
 from tool_shed.util import basic_util, encoding_util, hg_util, repository_util
 from tool_shed.util import shed_util_common as suc, tool_dependency_util
 from tool_shed.util import tool_util, xml_util
@@ -499,7 +499,7 @@ class InstallRepositoryManager(object):
         reinstalling an uninstalled repository.
         """
         shed_config_dict = self.app.toolbox.get_shed_config_dict_by_filename(shed_tool_conf)
-        tdtm = data_table_manager.ToolDataTableManager(self.app)
+        stdtm = ShedToolDataTableManager(self.app)
         irmm = InstalledRepositoryMetadataManager(app=self.app,
                                                   tpm=self.tpm,
                                                   repository=tool_shed_repository,
@@ -527,9 +527,9 @@ class InstallRepositoryManager(object):
                                                                 set_status=True)
         if 'sample_files' in irmm_metadata_dict:
             sample_files = irmm_metadata_dict.get('sample_files', [])
-            tool_index_sample_files = tdtm.get_tool_index_sample_files(sample_files)
+            tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files)
             tool_data_table_conf_filename, tool_data_table_elems = \
-                tdtm.install_tool_data_tables(tool_shed_repository, tool_index_sample_files)
+                stdtm.install_tool_data_tables(tool_shed_repository, tool_index_sample_files)
             if tool_data_table_elems:
                 self.app.tool_data_tables.add_new_entries_from_config_file(tool_data_table_conf_filename,
                                                                            self.app.config.shed_tool_data_path,
@@ -538,13 +538,13 @@ class InstallRepositoryManager(object):
         if 'tools' in irmm_metadata_dict:
             tool_panel_dict = self.tpm.generate_tool_panel_dict_for_new_install(irmm_metadata_dict['tools'], tool_section)
             sample_files = irmm_metadata_dict.get('sample_files', [])
-            tool_index_sample_files = tdtm.get_tool_index_sample_files(sample_files)
+            tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files)
             tool_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=tool_path)
             sample_files_copied = [str(s) for s in tool_index_sample_files]
             repository_tools_tups = irmm.get_repository_tools_tups()
             if repository_tools_tups:
                 # Handle missing data table entries for tool parameters that are dynamically generated select lists.
-                repository_tools_tups = tdtm.handle_missing_data_table_entry(relative_install_dir,
+                repository_tools_tups = stdtm.handle_missing_data_table_entry(relative_install_dir,
                                                                              tool_path,
                                                                              repository_tools_tups)
                 # Handle missing index files for tool parameters that are dynamically generated select lists.
diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py
index 17d1f3ebc3c..a352ce410c4 100644
--- a/lib/tool_shed/tools/data_table_manager.py
+++ b/lib/tool_shed/tools/data_table_manager.py
@@ -9,7 +9,7 @@ from tool_shed.util import hg_util, xml_util
 log = logging.getLogger(__name__)
 
 
-class ToolDataTableManager(object):
+class ShedToolDataTableManager(object):
 
     def __init__(self, app):
         self.tdtm = tdtm(app.config.shed_tool_data_path)
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index 4ab69d7eafe..2197c162728 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -7,8 +7,10 @@ from galaxy.tools import (
     parameters,
     Tool
 )
+from galaxy.tools.data import ToolDataTableManager
+from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import dynamic_options
-from tool_shed.tools import data_table_manager
+from tool_shed.tools.data_table_manager import ShedToolDataTableManager
 from tool_shed.util import (
     basic_util,
     hg_util,
@@ -24,7 +26,7 @@ class ToolValidator(object):
 
     def __init__(self, app):
         self.app = app
-        self.tdtm = data_table_manager.ToolDataTableManager(self.app)
+        self.stdtm = ShedToolDataTableManager(self.app)
 
     def can_use_tool_config_disk_file(self, repository, repo, file_path, changeset_revision):
         """
@@ -67,7 +69,7 @@ class ToolValidator(object):
                         sample_tool_data_table_conf = hg_util.get_config_from_disk('tool_data_table_conf.xml.sample', repo_dir)
                         if sample_tool_data_table_conf:
                             error, correction_msg = \
-                                self.tdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
+                                self.stdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
                                                                                   persist=False)
                             if error:
                                 invalid_files_and_errors_tups.append(('tool_data_table_conf.xml.sample', correction_msg))
@@ -204,7 +206,7 @@ class ToolValidator(object):
             if 'tool_data_table_conf.xml.sample' in sample_files:
                 # Load entries into the tool_data_tables if the tool requires them.
                 tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
-                error, message = self.tdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
+                error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
                                                                                    persist=False)
         tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath)
         message = self.concat_messages(message, message2)
@@ -224,7 +226,7 @@ class ToolValidator(object):
                 # Load entries into the tool_data_tables if the tool requires them.
                 tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
                 if tool_data_table_config:
-                    error, message = self.tdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
+                    error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
                                                                                        persist=False)
                     if error:
                         log.debug(message)
@@ -286,7 +288,7 @@ class ToolValidator(object):
         basic_util.remove_dir(work_dir)
         self.app.config.tool_data_path = original_tool_data_path
         # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
-        self.tdtm.reset_tool_data_tables()
+        self.stdtm.reset_tool_data_tables()
         return repository, tool, message
 
     def load_tool_from_config(self, repository_id, full_path):
diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py
index b73fdf01eae..803b50019c1 100644
--- a/lib/tool_shed/util/commit_util.py
+++ b/lib/tool_shed/util/commit_util.py
@@ -12,7 +12,7 @@ from sqlalchemy.sql.expression import null
 import tool_shed.repository_types.util as rt_util
 from galaxy.util import checkers
 from galaxy.util.path import safe_relpath
-from tool_shed.tools import data_table_manager
+from tool_shed.tools.data_table_manager import ShedToolDataTableManager
 from tool_shed.util import basic_util, hg_util, shed_util_common as suc
 
 if sys.version_info < (3, 3):
@@ -214,8 +214,8 @@ def handle_directory_changes(app, host, username, repository, full_path, filenam
             # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
             # by parsing the file and adding new entries to the in-memory app.tool_data_tables
             # dictionary.
-            tdtm = data_table_manager.ToolDataTableManager(app)
-            error, message = tdtm.handle_sample_tool_data_table_conf_file(filename_in_archive, persist=False)
+            stdtm = ShedToolDataTableManager(app)
+            error, message = stdtm.handle_sample_tool_data_table_conf_file(filename_in_archive, persist=False)
             if error:
                 return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
     hg_util.commit_changeset(repo.ui,

From b03f16ed49074ef98ba7ecd7ed9f70123e196ace Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 1 Dec 2017 10:09:33 +0100
Subject: [PATCH 08/23] Load tools directly from source and avoid loading via
 toolbox

---
 lib/tool_shed/metadata/metadata_generator.py |  6 +++++
 lib/tool_shed/tools/tool_validator.py        | 24 ++++++++++++++++----
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py
index 883ee58b1e8..aef41ffd81f 100644
--- a/lib/tool_shed/metadata/metadata_generator.py
+++ b/lib/tool_shed/metadata/metadata_generator.py
@@ -322,6 +322,8 @@ class MetadataGenerator(object):
             metadata_dict = {}
         readme_files = []
         invalid_tool_configs = []
+        old_tool_data_tables = self.app.tool_data_tables
+        old_tool_data_table_path = self.app.config.tool_data_path
         if self.resetting_all_metadata_on_repository:
             if not self.relative_install_dir:
                 raise Exception("The value of self.repository.repo_path must be set when resetting all metadata on a repository.")
@@ -477,8 +479,12 @@ class MetadataGenerator(object):
                 self.invalid_file_tups.append((rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, error_message))
         if invalid_tool_configs:
             metadata_dict['invalid_tools'] = invalid_tool_configs
+        else:
+            pass
         self.metadata_dict = metadata_dict
         basic_util.remove_dir(work_dir)
+        self.app.tool_data_tables = old_tool_data_tables
+        self.app.config.tool_data_path = old_tool_data_table_path
 
     def generate_package_dependency_metadata(self, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict):
         """
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index 2197c162728..16e8547a322 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -4,6 +4,8 @@ import os
 import tempfile
 
 from galaxy.tools import (
+    create_tool_from_source,
+    get_tool_source,
     parameters,
     Tool
 )
@@ -202,13 +204,14 @@ class ToolValidator(object):
         """
         message = ''
         sample_files = self.copy_disk_sample_files_to_dir(repo_files_dir, work_dir)
+        tool_data_table_config = None
         if sample_files:
             if 'tool_data_table_conf.xml.sample' in sample_files:
                 # Load entries into the tool_data_tables if the tool requires them.
                 tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
                 error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
                                                                                    persist=False)
-        tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath)
+        tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath, tool_data_table_config)
         message = self.concat_messages(message, message2)
         return tool, valid, message, sample_files
 
@@ -244,7 +247,6 @@ class ToolValidator(object):
         the received valid changeset revision and the first changeset revision in the repository,
         searching backwards.
         """
-        original_tool_data_path = self.app.config.tool_data_path
         repository = repository_util.get_repository_in_tool_shed(self.app, repository_id)
         repo_files_dir = repository.repo_path(self.app)
         repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_files_dir, create=False)
@@ -286,14 +288,25 @@ class ToolValidator(object):
                                                                        tool_config_filename,
                                                                        work_dir)
         basic_util.remove_dir(work_dir)
-        self.app.config.tool_data_path = original_tool_data_path
         # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
         self.stdtm.reset_tool_data_tables()
         return repository, tool, message
 
-    def load_tool_from_config(self, repository_id, full_path):
+    def load_tool_from_config(self, repository_id, full_path, tool_data_table_config=None):
+        tool_source = get_tool_source(
+            full_path,
+            enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False),
+            tool_location_fetcher=ToolLocationFetcher(),
+        )
+        old_tool_data_tables = self.app.tool_data_tables
+        old_tool_data_path = self.app.config.tool_data_path
+        tool_data_path = os.path.dirname(full_path)
+        self.app.config.tool_data_path = tool_data_path
         try:
-            tool = self.app.toolbox.load_tool(full_path, repository_id=repository_id, allow_code_files=False, use_cached=False)
+            self.app.tool_data_tables = ToolDataTableManager(tool_data_path)
+            self.app.tool_data_tables.add_new_entries_from_config_file(full_path, tool_data_path=tool_data_path, shed_tool_data_table_config='')
+            tool = create_tool_from_source(config_file=full_path, app=self.app, tool_source=tool_source, repository_id=repository_id)
+            # tool = self.app.toolbox.load_tool(full_path, repository_id=repository_id, allow_code_files=False, use_cached=False)
             valid = True
             error_message = None
         except KeyError as e:
@@ -303,6 +316,7 @@ class ToolValidator(object):
             error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct '
             error_message += 'this error.  '
         except Exception as e:
+            raise
             tool = None
             valid = False
             error_message = str(e)

From c0887d72f8d37ee7c608fa0e41d2455956e6791a Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 1 Dec 2017 14:02:25 +0100
Subject: [PATCH 09/23] Fix missing index file check

---
 lib/tool_shed/galaxy_install/install_manager.py | 2 +-
 lib/tool_shed/tools/data_table_manager.py       | 5 +++--
 lib/tool_shed/tools/tool_validator.py           | 7 ++++---
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 490c172b903..eded7c8a0d5 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -532,7 +532,7 @@ class InstallRepositoryManager(object):
                 stdtm.install_tool_data_tables(tool_shed_repository, tool_index_sample_files)
             if tool_data_table_elems:
                 self.app.tool_data_tables.add_new_entries_from_config_file(tool_data_table_conf_filename,
-                                                                           self.app.config.shed_tool_data_path,
+                                                                           None,
                                                                            self.app.config.shed_tool_data_table_config,
                                                                            persist=True)
         if 'tools' in irmm_metadata_dict:
diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py
index a352ce410c4..f29db7f8291 100644
--- a/lib/tool_shed/tools/data_table_manager.py
+++ b/lib/tool_shed/tools/data_table_manager.py
@@ -90,7 +90,7 @@ class ShedToolDataTableManager(object):
             self.reset_tool_data_tables()
         return repository_tools_tups
 
-    def handle_sample_tool_data_table_conf_file(self, filename, persist=False):
+    def handle_sample_tool_data_table_conf_file(self, filename, persist=False, tool_data_tables=None):
         """
         Parse the incoming filename and add new entries to the in-memory
         self.app.tool_data_tables dictionary.  If persist is True (should
@@ -100,8 +100,9 @@ class ShedToolDataTableManager(object):
         """
         error = False
         message = ''
+        tdtm = tool_data_tables or self.tdtm
         try:
-            new_table_elems, message = self.tdtm \
+            new_table_elems, message = tdtm \
                 .add_new_entries_from_config_file(config_filename=filename,
                                                   tool_data_path=self.app.config.shed_tool_data_path,
                                                   shed_tool_data_table_config=self.app.config.shed_tool_data_table_config,
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index 16e8547a322..c3c760c6de8 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -72,7 +72,8 @@ class ToolValidator(object):
                         if sample_tool_data_table_conf:
                             error, correction_msg = \
                                 self.stdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
-                                                                                  persist=False)
+                                                                                  persist=False,
+                                                                                  tool_data_tables=tool.app.tool_data_tables)
                             if error:
                                 invalid_files_and_errors_tups.append(('tool_data_table_conf.xml.sample', correction_msg))
                         else:
@@ -82,9 +83,9 @@ class ToolValidator(object):
                             invalid_tup = (tool_config_name, correction_msg)
                             if invalid_tup not in invalid_files_and_errors_tups:
                                 invalid_files_and_errors_tups.append(invalid_tup)
-                    if options.index_file or options.missing_index_file:
+                    if options.index_file or options.tool_data_table and options.tool_data_table.missing_index_file:
                         # Make sure the repository contains the required xxx.loc.sample file.
-                        index_file = options.index_file or options.missing_index_file
+                        index_file = options.index_file or options.tool_data_table.missing_index_file
                         index_file_name = basic_util.strip_path(index_file)
                         sample_found = False
                         for sample_file in sample_files:

From 55e458262b4184557842312c77f70dc1f5f3903c Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 09:30:31 +0100
Subject: [PATCH 10/23] Add miniapp to avoid manipulating global settings

---
 lib/galaxy/util/miniapp.py                    | 50 +++++++++++++++++++
 lib/tool_shed/metadata/metadata_generator.py  |  8 ++-
 .../metadata/repository_metadata_manager.py   |  1 -
 3 files changed, 53 insertions(+), 6 deletions(-)
 create mode 100644 lib/galaxy/util/miniapp.py

diff --git a/lib/galaxy/util/miniapp.py b/lib/galaxy/util/miniapp.py
new file mode 100644
index 00000000000..cf9c1f18521
--- /dev/null
+++ b/lib/galaxy/util/miniapp.py
@@ -0,0 +1,50 @@
+"""Create a smaller version of app for verifying tools"""
+import os
+import shutil
+import tempfile
+
+from contextlib import contextmanager
+
+from .bunch import Bunch
+from galaxy.datatypes.registry import Registry
+from galaxy.tools.data import ToolDataTableManager
+
+
+class MiniApp(object):
+    """Minimal App object for tool validation."""
+
+    def __init__(self, app_name, security, tool_data_path, shed_tool_data_path, tool_data_tables=None, registry=None):
+        self.name = app_name
+        self.security = security
+        self.config = Bunch()
+        self.config.tool_data_path = tool_data_path
+        self.config.shed_tool_data_path = shed_tool_data_path
+        _, self.config.tool_data_table_config = tempfile.mkstemp()
+        _, self.config.shed_tool_data_table_config = tempfile.mkstemp()
+        self.tool_data_tables = tool_data_tables
+        self.datatypes_registry=registry or Registry()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            os.remove(self.config.tool_data_table_config)
+            if self.config.shed_tool_data_table_config != self.config.tool_data_table_config:
+                os.remove(self.config.shed_tool_data_table_config)
+        except Exception:
+            pass
+
+    @contextmanager
+    @staticmethod
+    def from_app(app, work_dir=None):
+        cleanup = False
+        if not work_dir:
+            work_dir = tempfile.mkdtemp()
+            cleanup = True
+        tool_data_tables = ToolDataTableManager(work_dir)
+        app_name = app.name
+        yield MiniApp(app_name=app_name,
+                       security=app.security,
+                       tool_data_path=work_dir,
+                       shed_tool_data_path=work_dir,
+                       tool_data_tables=tool_data_tables)
+        if cleanup:
+            shutil.rmtree(work_dir, ignore_errors=True)
\ No newline at end of file
diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py
index aef41ffd81f..4d4160c16ae 100644
--- a/lib/tool_shed/metadata/metadata_generator.py
+++ b/lib/tool_shed/metadata/metadata_generator.py
@@ -6,6 +6,7 @@ import tempfile
 from sqlalchemy import and_
 
 from galaxy import util
+from galaxy.util.miniapp import MiniApp
 from galaxy.tools.data_manager.manager import DataManager
 from galaxy.tools.data import ToolDataTableManager
 from galaxy.tools.loader_directory import looks_like_a_tool
@@ -305,7 +306,6 @@ class MetadataGenerator(object):
         tool_data_table_conf.xml.sample file, in which case the entries should ultimately be
         persisted to the file referred to by self.app.config.shed_tool_data_table_config.
         """
-        tv = tool_validator.ToolValidator(self.app)
         if self.shed_config_dict is None:
             self.shed_config_dict = {}
         if self.updating_installed_repository:
@@ -322,8 +322,6 @@ class MetadataGenerator(object):
             metadata_dict = {}
         readme_files = []
         invalid_tool_configs = []
-        old_tool_data_tables = self.app.tool_data_tables
-        old_tool_data_table_path = self.app.config.tool_data_path
         if self.resetting_all_metadata_on_repository:
             if not self.relative_install_dir:
                 raise Exception("The value of self.repository.repo_path must be set when resetting all metadata on a repository.")
@@ -342,6 +340,8 @@ class MetadataGenerator(object):
             files_dir = self.relative_install_dir
             if self.shed_config_dict.get('tool_path'):
                 files_dir = os.path.join(self.shed_config_dict['tool_path'], files_dir)
+        app = MiniApp.from_app(app=self.app, work_dir=work_dir)
+        tv = tool_validator.ToolValidator(app)
         # Handle proprietary datatypes, if any.
         datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, files_dir)
         if datatypes_config:
@@ -483,8 +483,6 @@ class MetadataGenerator(object):
             pass
         self.metadata_dict = metadata_dict
         basic_util.remove_dir(work_dir)
-        self.app.tool_data_tables = old_tool_data_tables
-        self.app.config.tool_data_path = old_tool_data_table_path
 
     def generate_package_dependency_metadata(self, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict):
         """
diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py
index fb8c60a4f31..026ca56d4c4 100644
--- a/lib/tool_shed/metadata/repository_metadata_manager.py
+++ b/lib/tool_shed/metadata/repository_metadata_manager.py
@@ -990,7 +990,6 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator):
                                                                    self.metadata_dict)
             status = 'error'
         # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
-        self.app.tool_data_tables.data_tables = {}
         return message, status
 
     def set_repository_metadata_due_to_new_tip(self, host, content_alert_str=None, **kwd):

From ab21f303af8d53fe543974ed48e7d0614db2de22 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 10:36:42 +0100
Subject: [PATCH 11/23] Always use MiniApp when using ToolValidator

---
 lib/galaxy/util/miniapp.py                    |  37 ++-
 lib/galaxy/webapps/tool_shed/api/tools.py     |   9 +-
 .../tool_shed/controllers/repository.py       |  92 +++----
 .../webapps/tool_shed/controllers/upload.py   |   2 +-
 lib/tool_shed/metadata/metadata_generator.py  | 233 +++++++++---------
 lib/tool_shed/tools/tool_validator.py         |  12 +-
 lib/tool_shed/util/workflow_util.py           |  20 +-
 7 files changed, 209 insertions(+), 196 deletions(-)

diff --git a/lib/galaxy/util/miniapp.py b/lib/galaxy/util/miniapp.py
index cf9c1f18521..3eeb487b75a 100644
--- a/lib/galaxy/util/miniapp.py
+++ b/lib/galaxy/util/miniapp.py
@@ -5,24 +5,36 @@ import tempfile
 
 from contextlib import contextmanager
 
-from .bunch import Bunch
 from galaxy.datatypes.registry import Registry
 from galaxy.tools.data import ToolDataTableManager
+from .bunch import Bunch
 
 
 class MiniApp(object):
     """Minimal App object for tool validation."""
 
-    def __init__(self, app_name, security, tool_data_path, shed_tool_data_path, tool_data_tables=None, registry=None):
+    def __init__(self, app_name,
+                 security,
+                 model,
+                 tool_data_path,
+                 shed_tool_data_path,
+                 tool_data_tables=None,
+                 registry=None,
+                 hgweb_config_manager=None):
         self.name = app_name
         self.security = security
+        self.model = model
         self.config = Bunch()
         self.config.tool_data_path = tool_data_path
         self.config.shed_tool_data_path = shed_tool_data_path
         _, self.config.tool_data_table_config = tempfile.mkstemp()
         _, self.config.shed_tool_data_table_config = tempfile.mkstemp()
         self.tool_data_tables = tool_data_tables
-        self.datatypes_registry=registry or Registry()
+        self.datatypes_registry = registry or Registry()
+        self.hgweb_config_manager = hgweb_config_manager
+
+    def __enter__(self):
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         try:
@@ -32,19 +44,22 @@ class MiniApp(object):
         except Exception:
             pass
 
-    @contextmanager
     @staticmethod
+    @contextmanager
     def from_app(app, work_dir=None):
         cleanup = False
         if not work_dir:
             work_dir = tempfile.mkdtemp()
             cleanup = True
         tool_data_tables = ToolDataTableManager(work_dir)
-        app_name = app.name
-        yield MiniApp(app_name=app_name,
-                       security=app.security,
-                       tool_data_path=work_dir,
-                       shed_tool_data_path=work_dir,
-                       tool_data_tables=tool_data_tables)
+        with MiniApp(app_name=app.name,
+                     security=app.security,
+                     model=app.model,
+                     tool_data_path=work_dir,
+                     shed_tool_data_path=work_dir,
+                     tool_data_tables=tool_data_tables,
+                     hgweb_config_manager=getattr(app, 'hgweb_config_manager', None)
+                     ) as app:
+            yield app
         if cleanup:
-            shutil.rmtree(work_dir, ignore_errors=True)
\ No newline at end of file
+            shutil.rmtree(work_dir, ignore_errors=True)
diff --git a/lib/galaxy/webapps/tool_shed/api/tools.py b/lib/galaxy/webapps/tool_shed/api/tools.py
index b049e4b771c..bc0dd77b086 100644
--- a/lib/galaxy/webapps/tool_shed/api/tools.py
+++ b/lib/galaxy/webapps/tool_shed/api/tools.py
@@ -163,10 +163,11 @@ class ToolsController(BaseAPIController):
             trans.response.status = 404
             return {'status': 'error', 'message': message}
 
-        tv = tool_validator.ToolValidator(trans.app)
-        repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
-                                                                         changeset,
-                                                                         found_tool.tool_config)
+        with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
+            tv = tool_validator.ToolValidator(miniapp)
+            repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
+                                                                             changeset,
+                                                                             found_tool.tool_config)
         if message:
             status = 'error'
             return dict(message=message, status=status)
diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py
index 5a710847d8b..0f713bde4e3 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -864,10 +864,11 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         message = escape(kwd.get('message', ''))
         status = kwd.get('status', 'done')
         render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed')
-        tv = tool_validator.ToolValidator(trans.app)
-        repository, tool, message = tv.load_tool_from_changeset_revision(repository_id,
-                                                                         changeset_revision,
-                                                                         tool_config)
+        with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
+            tv = tool_validator.ToolValidator(miniapp)
+            repository, tool, message = tv.load_tool_from_changeset_revision(repository_id,
+                                                                             changeset_revision,
+                                                                             tool_config)
         if message:
             status = 'error'
         tool_state = tool_util.new_state(trans, tool, invalid=False)
@@ -1757,10 +1758,12 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
     def load_invalid_tool(self, trans, repository_id, tool_config, changeset_revision, **kwd):
         message = escape(kwd.get('message', ''))
         render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed')
-        tv = tool_validator.ToolValidator(trans.app)
-        repository, tool, error_message = tv.load_tool_from_changeset_revision(repository_id,
-                                                                               changeset_revision,
-                                                                               tool_config)
+
+        with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
+            tv = tool_validator.ToolValidator(miniapp)
+            repository, tool, error_message = tv.load_tool_from_changeset_revision(repository_id,
+                                                                                   changeset_revision,
+                                                                                   tool_config)
         tool_state = tool_util.new_state(trans, tool, invalid=True)
         invalid_file_tups = []
         if tool:
@@ -2891,42 +2894,43 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
             metadata = repository_metadata.metadata
             if metadata:
                 if 'tools' in metadata:
-                    tv = tool_validator.ToolValidator(trans.app)
-                    for tool_metadata_dict in metadata['tools']:
-                        if tool_metadata_dict['id'] == tool_id:
-                            work_dir = tempfile.mkdtemp()
-                            relative_path_to_tool_config = tool_metadata_dict['tool_config']
-                            guid = tool_metadata_dict['guid']
-                            full_path_to_tool_config = os.path.abspath(relative_path_to_tool_config)
-                            full_path_to_dir, tool_config_filename = os.path.split(full_path_to_tool_config)
-                            can_use_disk_file = tv.can_use_tool_config_disk_file(repository,
-                                                                                 repo,
-                                                                                 full_path_to_tool_config,
-                                                                                 changeset_revision)
-                            if can_use_disk_file:
-                                tool, valid, message, sample_files = \
-                                    tv.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
-                                                                                   repository_id,
-                                                                                   full_path_to_tool_config,
-                                                                                   work_dir)
-                                if message:
-                                    status = 'error'
-                            else:
-                                tool, message, sample_files = \
-                                    tv.handle_sample_files_and_load_tool_from_tmp_config(repo,
-                                                                                         repository_id,
-                                                                                         changeset_revision,
-                                                                                         tool_config_filename,
-                                                                                         work_dir)
-                                if message:
-                                    status = 'error'
-                            basic_util.remove_dir(work_dir)
-                            break
-                    if guid:
-                        tvm = tool_version_manager.ToolVersionManager(trans.app)
-                        tool_lineage = tvm.get_version_lineage_for_tool(repository_id,
-                                                                        repository_metadata,
-                                                                        guid)
+                    with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
+                        tv = tool_validator.ToolValidator(miniapp)
+                        for tool_metadata_dict in metadata['tools']:
+                            if tool_metadata_dict['id'] == tool_id:
+                                work_dir = tempfile.mkdtemp()
+                                relative_path_to_tool_config = tool_metadata_dict['tool_config']
+                                guid = tool_metadata_dict['guid']
+                                full_path_to_tool_config = os.path.abspath(relative_path_to_tool_config)
+                                full_path_to_dir, tool_config_filename = os.path.split(full_path_to_tool_config)
+                                can_use_disk_file = tv.can_use_tool_config_disk_file(repository,
+                                                                                     repo,
+                                                                                     full_path_to_tool_config,
+                                                                                     changeset_revision)
+                                if can_use_disk_file:
+                                    tool, valid, message, sample_files = \
+                                        tv.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
+                                                                                       repository_id,
+                                                                                       full_path_to_tool_config,
+                                                                                       work_dir)
+                                    if message:
+                                        status = 'error'
+                                else:
+                                    tool, message, sample_files = \
+                                        tv.handle_sample_files_and_load_tool_from_tmp_config(repo,
+                                                                                             repository_id,
+                                                                                             changeset_revision,
+                                                                                             tool_config_filename,
+                                                                                             work_dir)
+                                    if message:
+                                        status = 'error'
+                                basic_util.remove_dir(work_dir)
+                                break
+                        if guid:
+                            tvm = tool_version_manager.ToolVersionManager(trans.app)
+                            tool_lineage = tvm.get_version_lineage_for_tool(repository_id,
+                                                                            repository_metadata,
+                                                                            guid)
         else:
             repository_metadata_id = None
             metadata = None
diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py
index 7395035c0f9..10ce72680a6 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/upload.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py
@@ -17,6 +17,7 @@ from tool_shed.dependencies import attribute_handlers
 from tool_shed.galaxy_install import dependency_display
 from tool_shed.metadata import repository_metadata_manager
 from tool_shed.repository_types import util as rt_util
+from tool_shed.tools.data_table_manager import ShedToolDataTableManager
 from tool_shed.util import (
     basic_util,
     commit_util,
@@ -27,7 +28,6 @@ from tool_shed.util import (
     xml_util
 )
 from tool_shed.util.web_util import escape
-from tool_shed.tools.data_table_manager import ShedToolDataTableManager
 
 log = logging.getLogger(__name__)
 
diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py
index 4d4160c16ae..db6cda85f5e 100644
--- a/lib/tool_shed/metadata/metadata_generator.py
+++ b/lib/tool_shed/metadata/metadata_generator.py
@@ -6,11 +6,10 @@ import tempfile
 from sqlalchemy import and_
 
 from galaxy import util
-from galaxy.util.miniapp import MiniApp
 from galaxy.tools.data_manager.manager import DataManager
-from galaxy.tools.data import ToolDataTableManager
 from galaxy.tools.loader_directory import looks_like_a_tool
 from galaxy.tools.parser.interface import TestCollectionDef
+from galaxy.util.miniapp import MiniApp
 from galaxy.web import url_for
 from tool_shed.repository_types import util as rt_util
 from tool_shed.tools import tool_validator
@@ -340,125 +339,125 @@ class MetadataGenerator(object):
             files_dir = self.relative_install_dir
             if self.shed_config_dict.get('tool_path'):
                 files_dir = os.path.join(self.shed_config_dict['tool_path'], files_dir)
-        app = MiniApp.from_app(app=self.app, work_dir=work_dir)
-        tv = tool_validator.ToolValidator(app)
-        # Handle proprietary datatypes, if any.
-        datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, files_dir)
-        if datatypes_config:
-            metadata_dict = self.generate_datatypes_metadata(tv,
-                                                             files_dir,
-                                                             datatypes_config,
-                                                             metadata_dict)
-        # Get the relative path to all sample files included in the repository for storage in
-        # the repository's metadata.
-        sample_file_metadata_paths, sample_file_copy_paths = \
-            self.get_sample_files_from_disk(repository_files_dir=files_dir,
-                                            tool_path=self.shed_config_dict.get('tool_path'),
-                                            relative_install_dir=self.relative_install_dir)
-        if sample_file_metadata_paths:
-            metadata_dict['sample_files'] = sample_file_metadata_paths
-        # Copy all sample files included in the repository to a single directory location so we
-        # can load tools that depend on them.
-        for sample_file in sample_file_copy_paths:
-            tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir)
-            # If the list of sample files includes a tool_data_table_conf.xml.sample file, load
-            # its table elements into memory.
-            relative_path, filename = os.path.split(sample_file)
-            if filename == 'tool_data_table_conf.xml.sample':
-                # We create a new ToolDataTableManager to avoid adding entries to the app-wide
-                # tool data tables. This is only used for checking that the data table is valid.
-                tool_data_tables = ToolDataTableManager(work_dir)
-                new_table_elems, error_message = \
-                    tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
-                                                                      tool_data_path=work_dir,
-                                                                      shed_tool_data_table_config=work_dir,
-                                                                      persist=False)
-                if error_message:
-                    self.invalid_file_tups.append((filename, error_message))
-        for root, dirs, files in os.walk(files_dir):
-            if root.find('.hg') < 0 and root.find('hgrc') < 0:
-                if '.hg' in dirs:
-                    dirs.remove('.hg')
-                for name in files:
-                    # See if we have a repository dependencies defined.
-                    if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
-                        path_to_repository_dependencies_config = os.path.join(root, name)
-                        metadata_dict, error_message = \
-                            self.generate_repository_dependency_metadata(path_to_repository_dependencies_config,
-                                                                         metadata_dict)
-                        if error_message:
-                            self.invalid_file_tups.append((name, error_message))
-                    # See if we have one or more READ_ME files.
-                    elif name.lower() in readme_file_names:
-                        relative_path_to_readme = self.get_relative_path_to_repository_file(root,
-                                                                                            name,
-                                                                                            self.relative_install_dir,
-                                                                                            work_dir,
-                                                                                            self.shed_config_dict)
-                        readme_files.append(relative_path_to_readme)
-                    # See if we have a tool config.
-                    elif looks_like_a_tool(os.path.join(root, name), invalid_names=self.NOT_TOOL_CONFIGS):
-                        full_path = str(os.path.abspath(os.path.join(root, name)))  # why the str, seems very odd
-                        element_tree, error_message = xml_util.parse_xml(full_path)
-                        if element_tree is None:
-                            is_tool = False
-                        else:
-                            element_tree_root = element_tree.getroot()
-                            is_tool = element_tree_root.tag == 'tool'
-                        if is_tool:
-                            tool, valid, error_message = \
-                                tv.load_tool_from_config(self.app.security.encode_id(self.repository.id),
-                                                         full_path)
-                            if tool is None:
-                                if not valid:
-                                    invalid_tool_configs.append(name)
-                                    self.invalid_file_tups.append((name, error_message))
+        # Create a MiniApp object to load and validate tools, data tables and datatypes
+        with MiniApp.from_app(app=self.app, work_dir=work_dir) as miniapp:
+            tv = tool_validator.ToolValidator(miniapp)
+            # Handle proprietary datatypes, if any.
+            datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, files_dir)
+            if datatypes_config:
+                metadata_dict = self.generate_datatypes_metadata(tv,
+                                                                 files_dir,
+                                                                 datatypes_config,
+                                                                 metadata_dict)
+            # Get the relative path to all sample files included in the repository for storage in
+            # the repository's metadata.
+            sample_file_metadata_paths, sample_file_copy_paths = \
+                self.get_sample_files_from_disk(repository_files_dir=files_dir,
+                                                tool_path=self.shed_config_dict.get('tool_path'),
+                                                relative_install_dir=self.relative_install_dir)
+            if sample_file_metadata_paths:
+                metadata_dict['sample_files'] = sample_file_metadata_paths
+            # Copy all sample files included in the repository to a single directory location so we
+            # can load tools that depend on them.
+            for sample_file in sample_file_copy_paths:
+                tool_util.copy_sample_file(self.app, sample_file, dest_path=work_dir)
+                # If the list of sample files includes a tool_data_table_conf.xml.sample file, load
+                # its table elements into memory.
+                relative_path, filename = os.path.split(sample_file)
+                if filename == 'tool_data_table_conf.xml.sample':
+                    # We create a new ToolDataTableManager to avoid adding entries to the app-wide
+                    # tool data tables. This is only used for checking that the data table is valid.
+                    new_table_elems, error_message = \
+                        miniapp.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
+                                                                              tool_data_path=work_dir,
+                                                                              shed_tool_data_table_config=work_dir,
+                                                                              persist=False)
+                    if error_message:
+                        self.invalid_file_tups.append((filename, error_message))
+            for root, dirs, files in os.walk(files_dir):
+                if root.find('.hg') < 0 and root.find('hgrc') < 0:
+                    if '.hg' in dirs:
+                        dirs.remove('.hg')
+                    for name in files:
+                        # See if we have a repository dependencies defined.
+                        if name == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
+                            path_to_repository_dependencies_config = os.path.join(root, name)
+                            metadata_dict, error_message = \
+                                self.generate_repository_dependency_metadata(path_to_repository_dependencies_config,
+                                                                             metadata_dict)
+                            if error_message:
+                                self.invalid_file_tups.append((name, error_message))
+                        # See if we have one or more READ_ME files.
+                        elif name.lower() in readme_file_names:
+                            relative_path_to_readme = self.get_relative_path_to_repository_file(root,
+                                                                                                name,
+                                                                                                self.relative_install_dir,
+                                                                                                work_dir,
+                                                                                                self.shed_config_dict)
+                            readme_files.append(relative_path_to_readme)
+                        # See if we have a tool config.
+                        elif looks_like_a_tool(os.path.join(root, name), invalid_names=self.NOT_TOOL_CONFIGS):
+                            full_path = str(os.path.abspath(os.path.join(root, name)))  # why the str, seems very odd
+                            element_tree, error_message = xml_util.parse_xml(full_path)
+                            if element_tree is None:
+                                is_tool = False
                             else:
-                                invalid_files_and_errors_tups = \
-                                    tv.check_tool_input_params(files_dir,
-                                                               name,
-                                                               tool,
-                                                               sample_file_copy_paths)
-                                can_set_metadata = True
-                                for tup in invalid_files_and_errors_tups:
-                                    if name in tup:
-                                        can_set_metadata = False
+                                element_tree_root = element_tree.getroot()
+                                is_tool = element_tree_root.tag == 'tool'
+                            if is_tool:
+                                tool, valid, error_message = \
+                                    tv.load_tool_from_config(self.app.security.encode_id(self.repository.id),
+                                                             full_path)
+                                if tool is None:
+                                    if not valid:
                                         invalid_tool_configs.append(name)
-                                        break
-                                if can_set_metadata:
-                                    relative_path_to_tool_config = \
-                                        self.get_relative_path_to_repository_file(root,
-                                                                                  name,
-                                                                                  self.relative_install_dir,
-                                                                                  work_dir,
-                                                                                  self.shed_config_dict)
-                                    metadata_dict = self.generate_tool_metadata(relative_path_to_tool_config,
-                                                                                tool,
-                                                                                metadata_dict)
+                                        self.invalid_file_tups.append((name, error_message))
                                 else:
+                                    invalid_files_and_errors_tups = \
+                                        tv.check_tool_input_params(files_dir,
+                                                                   name,
+                                                                   tool,
+                                                                   sample_file_copy_paths)
+                                    can_set_metadata = True
                                     for tup in invalid_files_and_errors_tups:
-                                        self.invalid_file_tups.append(tup)
-                    # Find all exported workflows.
-                    elif name.endswith('.ga'):
-                        relative_path = os.path.join(root, name)
-                        if os.path.getsize(os.path.abspath(relative_path)) > 0:
-                            fp = open(relative_path, 'rb')
-                            workflow_text = fp.read()
-                            fp.close()
-                            if workflow_text:
-                                valid_exported_galaxy_workflow = True
-                                try:
-                                    exported_workflow_dict = json.loads(workflow_text)
-                                except Exception:
-                                    log.exception("Skipping file %s since it does not seem to be a valid exported Galaxy workflow",
-                                                  str(relative_path))
-                                    valid_exported_galaxy_workflow = False
-                            if valid_exported_galaxy_workflow and \
-                                'a_galaxy_workflow' in exported_workflow_dict and \
-                                    exported_workflow_dict['a_galaxy_workflow'] == 'true':
-                                metadata_dict = self.generate_workflow_metadata(relative_path,
-                                                                                exported_workflow_dict,
-                                                                                metadata_dict)
+                                        if name in tup:
+                                            can_set_metadata = False
+                                            invalid_tool_configs.append(name)
+                                            break
+                                    if can_set_metadata:
+                                        relative_path_to_tool_config = \
+                                            self.get_relative_path_to_repository_file(root,
+                                                                                      name,
+                                                                                      self.relative_install_dir,
+                                                                                      work_dir,
+                                                                                      self.shed_config_dict)
+                                        metadata_dict = self.generate_tool_metadata(relative_path_to_tool_config,
+                                                                                    tool,
+                                                                                    metadata_dict)
+                                    else:
+                                        for tup in invalid_files_and_errors_tups:
+                                            self.invalid_file_tups.append(tup)
+                        # Find all exported workflows.
+                        elif name.endswith('.ga'):
+                            relative_path = os.path.join(root, name)
+                            if os.path.getsize(os.path.abspath(relative_path)) > 0:
+                                fp = open(relative_path, 'rb')
+                                workflow_text = fp.read()
+                                fp.close()
+                                if workflow_text:
+                                    valid_exported_galaxy_workflow = True
+                                    try:
+                                        exported_workflow_dict = json.loads(workflow_text)
+                                    except Exception:
+                                        log.exception("Skipping file %s since it does not seem to be a valid exported Galaxy workflow",
+                                                      str(relative_path))
+                                        valid_exported_galaxy_workflow = False
+                                if valid_exported_galaxy_workflow and \
+                                    'a_galaxy_workflow' in exported_workflow_dict and \
+                                        exported_workflow_dict['a_galaxy_workflow'] == 'true':
+                                    metadata_dict = self.generate_workflow_metadata(relative_path,
+                                                                                    exported_workflow_dict,
+                                                                                    metadata_dict)
         # Handle any data manager entries
         data_manager_config = hg_util.get_config_from_disk(suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir)
         metadata_dict = self.generate_data_manager_metadata(files_dir,
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index c3c760c6de8..b2b47dc370e 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -212,7 +212,7 @@ class ToolValidator(object):
                 tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
                 error, message = self.stdtm.handle_sample_tool_data_table_conf_file(tool_data_table_config,
                                                                                    persist=False)
-        tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath, tool_data_table_config)
+        tool, valid, message2 = self.load_tool_from_config(repository_id, tool_config_filepath)
         message = self.concat_messages(message, message2)
         return tool, valid, message, sample_files
 
@@ -261,7 +261,6 @@ class ToolValidator(object):
                                                                tool_config_filepath,
                                                                changeset_revision)
         if can_use_disk_file:
-            # self.app.config.tool_data_path = work_dir
             tool, valid, message, sample_files = \
                 self.handle_sample_files_and_load_tool_from_disk(repo_files_dir,
                                                                  repository_id,
@@ -293,21 +292,14 @@ class ToolValidator(object):
         self.stdtm.reset_tool_data_tables()
         return repository, tool, message
 
-    def load_tool_from_config(self, repository_id, full_path, tool_data_table_config=None):
+    def load_tool_from_config(self, repository_id, full_path):
         tool_source = get_tool_source(
             full_path,
             enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False),
             tool_location_fetcher=ToolLocationFetcher(),
         )
-        old_tool_data_tables = self.app.tool_data_tables
-        old_tool_data_path = self.app.config.tool_data_path
-        tool_data_path = os.path.dirname(full_path)
-        self.app.config.tool_data_path = tool_data_path
         try:
-            self.app.tool_data_tables = ToolDataTableManager(tool_data_path)
-            self.app.tool_data_tables.add_new_entries_from_config_file(full_path, tool_data_path=tool_data_path, shed_tool_data_table_config='')
             tool = create_tool_from_source(config_file=full_path, app=self.app, tool_source=tool_source, repository_id=repository_id)
-            # tool = self.app.toolbox.load_tool(full_path, repository_id=repository_id, allow_code_files=False, use_cached=False)
             valid = True
             error_message = None
         except KeyError as e:
diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py
index 0a511c49f1c..ac13a327284 100644
--- a/lib/tool_shed/util/workflow_util.py
+++ b/lib/tool_shed/util/workflow_util.py
@@ -7,6 +7,7 @@ import os
 
 import galaxy.tools
 import galaxy.tools.parameters
+from galaxy.util.miniapp import MiniApp
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.workflow.modules import (
     module_types,
@@ -36,17 +37,18 @@ class RepoToolModule(ToolModule):
         self.tool = None
         self.errors = None
         self.tool_version = None
-        self.tv = tool_validator.ToolValidator(trans.app)
         if trans.webapp.name == 'tool_shed':
             # We're in the tool shed.
-            for tool_dict in tools_metadata:
-                if self.tool_id in [tool_dict['id'], tool_dict['guid']]:
-                    repository, self.tool, message = self.tv.load_tool_from_changeset_revision(repository_id,
-                                                                                               changeset_revision,
-                                                                                               tool_dict['tool_config'])
-                    if message and self.tool is None:
-                        self.errors = 'unavailable'
-                    break
+            with MiniApp.from_app(trans.app) as miniapp:
+                tv = tool_validator.ToolValidator(miniapp)
+                for tool_dict in tools_metadata:
+                    if self.tool_id in [tool_dict['id'], tool_dict['guid']]:
+                        repository, self.tool, message = tv.load_tool_from_changeset_revision(repository_id,
+                                                                                              changeset_revision,
+                                                                                              tool_dict['tool_config'])
+                        if message and self.tool is None:
+                            self.errors = 'unavailable'
+                        break
         else:
             # We're in Galaxy.
             self.tool = trans.app.toolbox.get_tool(self.tool_id, tool_version=self.tool_version)

From f08f4733050922c4177d84ec6f2550dd53cc8fcb Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 11:30:08 +0100
Subject: [PATCH 12/23] Only parse citations if app has citations_manager

---
 lib/galaxy/tools/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 4d872d4fc60..73aab82bcbe 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -940,9 +940,10 @@ class Tool(object, Dictifiable):
         for citation_elem in citations_elem:
             if citation_elem.tag != "citation":
                 pass
-            citation = self.app.citations_manager.parse_citation(citation_elem, self.tool_dir)
-            if citation:
-                citations.append(citation)
+            if hasattr(self.app, 'citations_manager'):
+                citation = self.app.citations_manager.parse_citation(citation_elem, self.tool_dir)
+                if citation:
+                    citations.append(citation)
         return citations
 
     def parse_input_elem(self, page_source, enctypes, context=None):

From d3261096c831a8539aeeae5ccf4e6734593f91a1 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 14:10:38 +0100
Subject: [PATCH 13/23] Fix linting issues and fix data manager installation

---
 lib/galaxy/util/miniapp.py            | 7 ++++++-
 lib/tool_shed/tools/tool_validator.py | 1 -
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/util/miniapp.py b/lib/galaxy/util/miniapp.py
index 3eeb487b75a..f3bc0ff2b41 100644
--- a/lib/galaxy/util/miniapp.py
+++ b/lib/galaxy/util/miniapp.py
@@ -2,11 +2,11 @@
 import os
 import shutil
 import tempfile
-
 from contextlib import contextmanager
 
 from galaxy.datatypes.registry import Registry
 from galaxy.tools.data import ToolDataTableManager
+from galaxy.util.dbkeys import GenomeBuilds
 from .bunch import Bunch
 
 
@@ -32,12 +32,17 @@ class MiniApp(object):
         self.tool_data_tables = tool_data_tables
         self.datatypes_registry = registry or Registry()
         self.hgweb_config_manager = hgweb_config_manager
+        _, self.config.len_file_path = tempfile.mkstemp()
+        self.config.builds_file_path = tempfile.mkdtemp()
+        self.genome_builds = GenomeBuilds(self)
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         try:
+            shutil.rmtree(self.config.builds_file_path, ignore_errors=True)
+            os.remove(self.config.len_file_path)
             os.remove(self.config.tool_data_table_config)
             if self.config.shed_tool_data_table_config != self.config.tool_data_table_config:
                 os.remove(self.config.shed_tool_data_table_config)
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index b2b47dc370e..602a6ba9fa9 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -9,7 +9,6 @@ from galaxy.tools import (
     parameters,
     Tool
 )
-from galaxy.tools.data import ToolDataTableManager
 from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import dynamic_options
 from tool_shed.tools.data_table_manager import ShedToolDataTableManager

From c1630c244fc4700998e0d23130f7120344b303b2 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 20:12:20 +0100
Subject: [PATCH 14/23] Move and rename MiniApp to ValidationContext

This better describes what it used for and since it is tool specific, fits
better here that in utils. Thanks @jmchilton for the suggestion.
---
 .../repositories/validationcontext.py}        | 22 +++++++--------
 lib/galaxy/webapps/tool_shed/api/tools.py     |  5 ++--
 .../tool_shed/controllers/repository.py       | 27 ++++++++++---------
 lib/tool_shed/metadata/metadata_generator.py  | 16 +++++------
 lib/tool_shed/util/workflow_util.py           |  6 ++---
 5 files changed, 39 insertions(+), 37 deletions(-)
 rename lib/galaxy/{util/miniapp.py => tools/repositories/validationcontext.py} (77%)

diff --git a/lib/galaxy/util/miniapp.py b/lib/galaxy/tools/repositories/validationcontext.py
similarity index 77%
rename from lib/galaxy/util/miniapp.py
rename to lib/galaxy/tools/repositories/validationcontext.py
index f3bc0ff2b41..8f47533c246 100644
--- a/lib/galaxy/util/miniapp.py
+++ b/lib/galaxy/tools/repositories/validationcontext.py
@@ -1,4 +1,4 @@
-"""Create a smaller version of app for verifying tools"""
+"""Provides a subset of app for verifying tools."""
 import os
 import shutil
 import tempfile
@@ -6,11 +6,11 @@ from contextlib import contextmanager
 
 from galaxy.datatypes.registry import Registry
 from galaxy.tools.data import ToolDataTableManager
+from galaxy.util.bunch import Bunch
 from galaxy.util.dbkeys import GenomeBuilds
-from .bunch import Bunch
 
 
-class MiniApp(object):
+class ValidationContext(object):
     """Minimal App object for tool validation."""
 
     def __init__(self, app_name,
@@ -57,14 +57,14 @@ class MiniApp(object):
             work_dir = tempfile.mkdtemp()
             cleanup = True
         tool_data_tables = ToolDataTableManager(work_dir)
-        with MiniApp(app_name=app.name,
-                     security=app.security,
-                     model=app.model,
-                     tool_data_path=work_dir,
-                     shed_tool_data_path=work_dir,
-                     tool_data_tables=tool_data_tables,
-                     hgweb_config_manager=getattr(app, 'hgweb_config_manager', None)
-                     ) as app:
+        with ValidationContext(app_name=app.name,
+                               security=app.security,
+                               model=app.model,
+                               tool_data_path=work_dir,
+                               shed_tool_data_path=work_dir,
+                               tool_data_tables=tool_data_tables,
+                               hgweb_config_manager=getattr(app, 'hgweb_config_manager', None)
+                               ) as app:
             yield app
         if cleanup:
             shutil.rmtree(work_dir, ignore_errors=True)
diff --git a/lib/galaxy/webapps/tool_shed/api/tools.py b/lib/galaxy/webapps/tool_shed/api/tools.py
index bc0dd77b086..e22274b0ceb 100644
--- a/lib/galaxy/webapps/tool_shed/api/tools.py
+++ b/lib/galaxy/webapps/tool_shed/api/tools.py
@@ -12,6 +12,7 @@ from galaxy.exceptions import (
     RequestParameterInvalidException
 )
 from galaxy.tools.parameters import params_to_strings
+from galaxy.tools.repositories import ValidationContext
 from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
 from galaxy.web.base.controller import BaseAPIController
 from galaxy.webapps.tool_shed.search.tool_search import ToolSearch
@@ -163,8 +164,8 @@ class ToolsController(BaseAPIController):
             trans.response.status = 404
             return {'status': 'error', 'message': message}
 
-        with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
-            tv = tool_validator.ToolValidator(miniapp)
+        with ValidationContext.from_app(trans.app) as validation_context:
+            tv = tool_validator.ToolValidator(validation_context)
             repository, tool, message = tv.load_tool_from_changeset_revision(tsr_id,
                                                                              changeset,
                                                                              found_tool.tool_config)
diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py
index 0f713bde4e3..427856037c7 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -18,6 +18,7 @@ from galaxy import (
     util,
     web
 )
+from galaxy.tools.repositories import ValidationContext
 from galaxy.web.base.controller import BaseUIController
 from galaxy.web.form_builder import CheckboxField
 from galaxy.web.framework.helpers import grids
@@ -864,8 +865,8 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         message = escape(kwd.get('message', ''))
         status = kwd.get('status', 'done')
         render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed')
-        with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
-            tv = tool_validator.ToolValidator(miniapp)
+        with ValidationContext.from_app(trans.app) as validation_context:
+            tv = tool_validator.ToolValidator(validation_context)
             repository, tool, message = tv.load_tool_from_changeset_revision(repository_id,
                                                                              changeset_revision,
                                                                              tool_config)
@@ -1759,18 +1760,18 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
         message = escape(kwd.get('message', ''))
         render_repository_actions_for = kwd.get('render_repository_actions_for', 'tool_shed')
 
-        with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
-            tv = tool_validator.ToolValidator(miniapp)
+        with ValidationContext.from_app(trans.app) as validation_context:
+            tv = tool_validator.ToolValidator(validation_context)
             repository, tool, error_message = tv.load_tool_from_changeset_revision(repository_id,
                                                                                    changeset_revision,
                                                                                    tool_config)
-        tool_state = tool_util.new_state(trans, tool, invalid=True)
-        invalid_file_tups = []
-        if tool:
-            invalid_file_tups = tv.check_tool_input_params(repository.repo_path(trans.app),
-                                                           tool_config,
-                                                           tool,
-                                                           [])
+            tool_state = tool_util.new_state(trans, tool, invalid=True)
+            invalid_file_tups = []
+            if tool:
+                invalid_file_tups = tv.check_tool_input_params(repository.repo_path(trans.app),
+                                                               tool_config,
+                                                               tool,
+                                                               [])
         if invalid_file_tups:
             message = tool_util.generate_message_for_invalid_tools(trans.app,
                                                                    invalid_file_tups,
@@ -2894,8 +2895,8 @@ class RepositoryController(BaseUIController, ratings_util.ItemRatings):
             metadata = repository_metadata.metadata
             if metadata:
                 if 'tools' in metadata:
-                    with util.miniapp.MiniApp.from_app(trans.app) as miniapp:
-                        tv = tool_validator.ToolValidator(miniapp)
+                    with ValidationContext.from_app(trans.app) as validation_context:
+                        tv = tool_validator.ToolValidator(validation_context)
                         for tool_metadata_dict in metadata['tools']:
                             if tool_metadata_dict['id'] == tool_id:
                                 work_dir = tempfile.mkdtemp()
diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py
index db6cda85f5e..80d5e5af87d 100644
--- a/lib/tool_shed/metadata/metadata_generator.py
+++ b/lib/tool_shed/metadata/metadata_generator.py
@@ -9,7 +9,7 @@ from galaxy import util
 from galaxy.tools.data_manager.manager import DataManager
 from galaxy.tools.loader_directory import looks_like_a_tool
 from galaxy.tools.parser.interface import TestCollectionDef
-from galaxy.util.miniapp import MiniApp
+from galaxy.tools.repositories import ValidationContext
 from galaxy.web import url_for
 from tool_shed.repository_types import util as rt_util
 from tool_shed.tools import tool_validator
@@ -339,9 +339,9 @@ class MetadataGenerator(object):
             files_dir = self.relative_install_dir
             if self.shed_config_dict.get('tool_path'):
                 files_dir = os.path.join(self.shed_config_dict['tool_path'], files_dir)
-        # Create a MiniApp object to load and validate tools, data tables and datatypes
-        with MiniApp.from_app(app=self.app, work_dir=work_dir) as miniapp:
-            tv = tool_validator.ToolValidator(miniapp)
+        # Create ValidationContext to load and validate tools, data tables and datatypes
+        with ValidationContext.from_app(app=self.app, work_dir=work_dir) as validation_context:
+            tv = tool_validator.ToolValidator(validation_context)
             # Handle proprietary datatypes, if any.
             datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, files_dir)
             if datatypes_config:
@@ -368,10 +368,10 @@ class MetadataGenerator(object):
                     # We create a new ToolDataTableManager to avoid adding entries to the app-wide
                     # tool data tables. This is only used for checking that the data table is valid.
                     new_table_elems, error_message = \
-                        miniapp.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
-                                                                              tool_data_path=work_dir,
-                                                                              shed_tool_data_table_config=work_dir,
-                                                                              persist=False)
+                        validation_context.tool_data_tables.add_new_entries_from_config_file(config_filename=sample_file,
+                                                                                             tool_data_path=work_dir,
+                                                                                             shed_tool_data_table_config=work_dir,
+                                                                                             persist=False)
                     if error_message:
                         self.invalid_file_tups.append((filename, error_message))
             for root, dirs, files in os.walk(files_dir):
diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py
index ac13a327284..6a1a9eb49d6 100644
--- a/lib/tool_shed/util/workflow_util.py
+++ b/lib/tool_shed/util/workflow_util.py
@@ -7,7 +7,7 @@ import os
 
 import galaxy.tools
 import galaxy.tools.parameters
-from galaxy.util.miniapp import MiniApp
+from galaxy.tools.repositories import ValidationConext
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.workflow.modules import (
     module_types,
@@ -39,8 +39,8 @@ class RepoToolModule(ToolModule):
         self.tool_version = None
         if trans.webapp.name == 'tool_shed':
             # We're in the tool shed.
-            with MiniApp.from_app(trans.app) as miniapp:
-                tv = tool_validator.ToolValidator(miniapp)
+            with ValidationConext.from_app(trans.app) as validation_context:
+                tv = tool_validator.ToolValidator(validation_context)
                 for tool_dict in tools_metadata:
                     if self.tool_id in [tool_dict['id'], tool_dict['guid']]:
                         repository, self.tool, message = tv.load_tool_from_changeset_revision(repository_id,

From 93442f621bea51bd242f2b67e7d201bc409f9fe9 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 21:56:51 +0100
Subject: [PATCH 15/23] Move ValidationContext to right path

---
 .../validationcontext.py => repositories.py}      |  0
 lib/tool_shed/metadata/metadata_generator.py      | 15 +--------------
 lib/tool_shed/util/workflow_util.py               |  4 ++--
 3 files changed, 3 insertions(+), 16 deletions(-)
 rename lib/galaxy/tools/{repositories/validationcontext.py => repositories.py} (100%)

diff --git a/lib/galaxy/tools/repositories/validationcontext.py b/lib/galaxy/tools/repositories.py
similarity index 100%
rename from lib/galaxy/tools/repositories/validationcontext.py
rename to lib/galaxy/tools/repositories.py
diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py
index 80d5e5af87d..490e83490b9 100644
--- a/lib/tool_shed/metadata/metadata_generator.py
+++ b/lib/tool_shed/metadata/metadata_generator.py
@@ -478,8 +478,6 @@ class MetadataGenerator(object):
                 self.invalid_file_tups.append((rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, error_message))
         if invalid_tool_configs:
             metadata_dict['invalid_tools'] = invalid_tool_configs
-        else:
-            pass
         self.metadata_dict = metadata_dict
         basic_util.remove_dir(work_dir)
 
@@ -574,7 +572,6 @@ class MetadataGenerator(object):
         Generate a repository dependencies dictionary based on valid information defined in the received
         repository_dependencies_config.  This method is called from the tool shed as well as from Galaxy.
         """
-        error_message = ''
         # Make sure we're looking at a valid repository_dependencies.xml file.
         tree, error_message = xml_util.parse_xml(repository_dependencies_config)
         if tree is None:
@@ -824,6 +821,7 @@ class MetadataGenerator(object):
         return relative_path_to_file
 
     def get_sample_files_from_disk(self, repository_files_dir, tool_path=None, relative_install_dir=None):
+        work_dir = ''
         if self.resetting_all_metadata_on_repository:
             # Keep track of the location where the repository is temporarily cloned so that we can strip
             # it when setting metadata.
@@ -895,8 +893,6 @@ class MetadataGenerator(object):
             # Default to the current tool shed.
             toolshed = str(url_for('/', qualified=True)).rstrip('/')
             repository_dependency_tup[0] = toolshed
-        user = None
-        repository = None
         toolshed = common_util.remove_protocol_from_tool_shed_url(toolshed)
         if self.app.name == 'galaxy':
             # We're in Galaxy.  We reach here when we're generating the metadata for a tool
@@ -1094,15 +1090,6 @@ class MetadataGenerator(object):
         else:
             repository_dependencies_dict = metadata.get('invalid_repository_dependencies', None)
         for repository_dependency_tup in repository_dependency_tups:
-            if is_valid:
-                tool_shed, name, owner, changeset_revision, \
-                    prior_installation_required, \
-                    only_if_compiling_contained_td = repository_dependency_tup
-            else:
-                tool_shed, name, owner, changeset_revision, \
-                    prior_installation_required, \
-                    only_if_compiling_contained_td, error_message = \
-                    repository_dependency_tup
             if repository_dependencies_dict:
                 repository_dependencies = repository_dependencies_dict.get('repository_dependencies', [])
                 for repository_dependency_tup in repository_dependency_tups:
diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py
index 6a1a9eb49d6..38a7edd8f3e 100644
--- a/lib/tool_shed/util/workflow_util.py
+++ b/lib/tool_shed/util/workflow_util.py
@@ -7,7 +7,7 @@ import os
 
 import galaxy.tools
 import galaxy.tools.parameters
-from galaxy.tools.repositories import ValidationConext
+from galaxy.tools.repositories import ValidationContext
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.workflow.modules import (
     module_types,
@@ -39,7 +39,7 @@ class RepoToolModule(ToolModule):
         self.tool_version = None
         if trans.webapp.name == 'tool_shed':
             # We're in the tool shed.
-            with ValidationConext.from_app(trans.app) as validation_context:
+            with ValidationContext.from_app(trans.app) as validation_context:
                 tv = tool_validator.ToolValidator(validation_context)
                 for tool_dict in tools_metadata:
                     if self.tool_id in [tool_dict['id'], tool_dict['guid']]:

From 482c9cce76c56bee840a87f25b0a17b8937444f5 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sat, 2 Dec 2017 21:58:33 +0100
Subject: [PATCH 16/23] Remove more unused variables

---
 lib/tool_shed/tools/tool_validator.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index 602a6ba9fa9..f5b46bb2b7c 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -58,7 +58,6 @@ class ToolValidator(object):
         generated using external data files to make sure the files exist.
         """
         invalid_files_and_errors_tups = []
-        correction_msg = ''
         for input_param in tool.input_params:
             if isinstance(input_param, parameters.basic.SelectToolParameter) and input_param.is_dynamic:
                 # If the tool refers to .loc files or requires an entry in the tool_data_table_conf.xml,
@@ -224,7 +223,6 @@ class ToolValidator(object):
         # intended to help handle sample files that are in the manifest, but have been deleted from disk.
         sample_files, deleted_sample_files = self.get_list_of_copied_sample_files(repo, ctx, dir=work_dir)
         if sample_files:
-            # self.app.config.tool_data_path = work_dir
             if 'tool_data_table_conf.xml.sample' in sample_files:
                 # Load entries into the tool_data_tables if the tool requires them.
                 tool_data_table_config = os.path.join(work_dir, 'tool_data_table_conf.xml')
@@ -250,9 +248,6 @@ class ToolValidator(object):
         repository = repository_util.get_repository_in_tool_shed(self.app, repository_id)
         repo_files_dir = repository.repo_path(self.app)
         repo = hg_util.get_repo_for_repository(self.app, repository=None, repo_path=repo_files_dir, create=False)
-        message = ''
-        tool = None
-        can_use_disk_file = False
         tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository(repo_files_dir, tool_config_filename)
         work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr")
         can_use_disk_file = self.can_use_tool_config_disk_file(repository,
@@ -308,7 +303,6 @@ class ToolValidator(object):
             error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct '
             error_message += 'this error.  '
         except Exception as e:
-            raise
             tool = None
             valid = False
             error_message = str(e)

From db88175b9fffe76cfd9013a6e6d6cfb9c573c354 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 09:47:07 +0100
Subject: [PATCH 17/23] Use ValidationContext data tables instead of separately
 passing in relevant data tables

---
 lib/tool_shed/tools/data_table_manager.py | 11 ++++-------
 lib/tool_shed/tools/tool_validator.py     |  3 +--
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py
index f29db7f8291..be98f124d45 100644
--- a/lib/tool_shed/tools/data_table_manager.py
+++ b/lib/tool_shed/tools/data_table_manager.py
@@ -12,7 +12,6 @@ log = logging.getLogger(__name__)
 class ShedToolDataTableManager(object):
 
     def __init__(self, app):
-        self.tdtm = tdtm(app.config.shed_tool_data_path)
         self.app = app
 
     def generate_repository_info_elem(self, tool_shed, repository_name, changeset_revision, owner,
@@ -90,7 +89,7 @@ class ShedToolDataTableManager(object):
             self.reset_tool_data_tables()
         return repository_tools_tups
 
-    def handle_sample_tool_data_table_conf_file(self, filename, persist=False, tool_data_tables=None):
+    def handle_sample_tool_data_table_conf_file(self, filename, persist=False):
         """
         Parse the incoming filename and add new entries to the in-memory
         self.app.tool_data_tables dictionary.  If persist is True (should
@@ -99,10 +98,8 @@ class ShedToolDataTableManager(object):
         file on disk.
         """
         error = False
-        message = ''
-        tdtm = tool_data_tables or self.tdtm
         try:
-            new_table_elems, message = tdtm \
+            new_table_elems, message = self.app.tool_data_tables \
                 .add_new_entries_from_config_file(config_filename=filename,
                                                   tool_data_path=self.app.config.shed_tool_data_path,
                                                   shed_tool_data_table_config=self.app.config.shed_tool_data_table_config,
@@ -170,9 +167,9 @@ class ShedToolDataTableManager(object):
             # Remove old data_table
             os.unlink(tool_data_table_conf_filename)
             # Persist new data_table content.
-            self.tdtm.to_xml_file(tool_data_table_conf_filename, elems)
+            self.app.tool_data_tables.to_xml_file(tool_data_table_conf_filename, elems)
         return tool_data_table_conf_filename, elems
 
     def reset_tool_data_tables(self):
         # Reset the tool_data_tables to an empty dictionary.
-        self.tdtm.data_tables = {}
+        self.app.tool_data_tables.data_tables = {}
diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py
index f5b46bb2b7c..0cf3485059f 100644
--- a/lib/tool_shed/tools/tool_validator.py
+++ b/lib/tool_shed/tools/tool_validator.py
@@ -70,8 +70,7 @@ class ToolValidator(object):
                         if sample_tool_data_table_conf:
                             error, correction_msg = \
                                 self.stdtm.handle_sample_tool_data_table_conf_file(sample_tool_data_table_conf,
-                                                                                  persist=False,
-                                                                                  tool_data_tables=tool.app.tool_data_tables)
+                                                                                  persist=False)
                             if error:
                                 invalid_files_and_errors_tups.append(('tool_data_table_conf.xml.sample', correction_msg))
                         else:

From fd4737d0ecade2dd3f13d51756f90899920ef2a4 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 09:48:23 +0100
Subject: [PATCH 18/23] Don't reload tools, new indexes and data tables are
 recognized without a reload

---
 lib/tool_shed/tools/data_table_manager.py | 3 ---
 lib/tool_shed/util/tool_util.py           | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py
index be98f124d45..cf5dd21df2f 100644
--- a/lib/tool_shed/tools/data_table_manager.py
+++ b/lib/tool_shed/tools/data_table_manager.py
@@ -82,9 +82,6 @@ class ShedToolDataTableManager(object):
                 if error:
                     # TODO: Do more here than logging an exception.
                     log.debug(message)
-            # Reload the tool into the local list of repository_tools_tups.
-            repository_tool = self.app.toolbox.load_tool(os.path.join(tool_path, tup_path), guid=guid, use_cached=False)
-            repository_tools_tups[index] = (tup_path, guid, repository_tool)
             # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
             self.reset_tool_data_tables()
         return repository_tools_tups
diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py
index 958a299bd73..42dc7132d44 100644
--- a/lib/tool_shed/util/tool_util.py
+++ b/lib/tool_shed/util/tool_util.py
@@ -164,9 +164,6 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup
                             options.tool_data_table.handle_found_index_file(options.missing_index_file)
                         sample_files_copied.append(options.missing_index_file)
                         break
-        # Reload the tool into the local list of repository_tools_tups.
-        repository_tool = app.toolbox.load_tool(os.path.join(tool_path, tup_path), guid=guid, use_cached=False)
-        repository_tools_tups[index] = (tup_path, guid, repository_tool)
     return repository_tools_tups, sample_files_copied
 
 

From f97774be82108d94fecc29dcf98f1fe833a4ece0 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 09:57:25 +0100
Subject: [PATCH 19/23] Change ValidationContext.build_file_path to be file

Also simplify cleanup code.
---
 lib/galaxy/tools/repositories.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/tools/repositories.py b/lib/galaxy/tools/repositories.py
index 8f47533c246..5bc35faef62 100644
--- a/lib/galaxy/tools/repositories.py
+++ b/lib/galaxy/tools/repositories.py
@@ -33,21 +33,22 @@ class ValidationContext(object):
         self.datatypes_registry = registry or Registry()
         self.hgweb_config_manager = hgweb_config_manager
         _, self.config.len_file_path = tempfile.mkstemp()
-        self.config.builds_file_path = tempfile.mkdtemp()
+        _, self.config.builds_file_path = tempfile.mkstemp()
         self.genome_builds = GenomeBuilds(self)
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            shutil.rmtree(self.config.builds_file_path, ignore_errors=True)
-            os.remove(self.config.len_file_path)
-            os.remove(self.config.tool_data_table_config)
-            if self.config.shed_tool_data_table_config != self.config.tool_data_table_config:
-                os.remove(self.config.shed_tool_data_table_config)
-        except Exception:
-            pass
+        cleanup_paths = {self.config.builds_file_path,
+                         self.config.len_file_path,
+                         self.config.tool_data_table_config,
+                         self.config.shed_tool_data_table_config}
+        for path in cleanup_paths:
+           try:
+               os.remove(path)
+           except Exception:
+               pass
 
     @staticmethod
     @contextmanager

From 1b48ff3dd15ebde13382afdb41cfb200a3c883cc Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 10:24:09 +0100
Subject: [PATCH 20/23] Skip format parsing if datatypes registry contains no
 datatypes

---
 lib/galaxy/tools/parameters/basic.py | 33 ++++++++++++++++------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py
index 76faccdee44..6022318a2c2 100644
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -34,6 +34,7 @@ from ..parameters import (
     history_query
 )
 from ..parser import get_input_source as ensure_input_source
+from ..repositories import ValidationContext
 
 log = logging.getLogger(__name__)
 
@@ -1435,7 +1436,7 @@ class BaseDataToolParameter(ToolParameter):
         super(BaseDataToolParameter, self).__init__(tool, input_source)
         self.refresh_on_change = True
 
-    def _datatypes_registery(self, trans, tool):
+    def _datatypes_registry(self, trans, tool):
         # Find datatypes_registry
         if tool is None:
             if trans:
@@ -1448,22 +1449,26 @@ class BaseDataToolParameter(ToolParameter):
                 datatypes_registry = galaxy.datatypes.registry.Registry()
                 datatypes_registry.load_datatypes()
         else:
-            datatypes_registry = tool.app.datatypes_registry
+            if isinstance(tool.app, ValidationContext):
+                datatypes_registry = {}
+            else:
+                datatypes_registry = tool.app.datatypes_registry
         return datatypes_registry
 
     def _parse_formats(self, trans, tool, input_source):
-        datatypes_registry = self._datatypes_registery(trans, tool)
-
+        datatypes_registry = self._datatypes_registry(trans, tool)
+        formats = []
         # Build list of classes for supported data formats
         self.extensions = input_source.get('format', 'data').split(",")
         normalized_extensions = [extension.strip().lower() for extension in self.extensions]
-        formats = []
-        for extension in normalized_extensions:
-            datatype = datatypes_registry.get_datatype_by_extension(extension)
-            if datatype is not None:
-                formats.append(datatype)
-            else:
-                log.warning("Datatype class not found for extension '%s', which is used in the 'format' attribute of parameter '%s'" % (extension, self.name))
+        if datatypes_registry:
+            # Skip during validation since no datatypes are loaded
+            for extension in normalized_extensions:
+                datatype = datatypes_registry.get_datatype_by_extension(extension)
+                if datatype is not None:
+                    formats.append(datatype)
+                else:
+                    log.warning("Datatype class not found for extension '%s', which is used in the 'format' attribute of parameter '%s'" % (extension, self.name))
         self.formats = formats
 
     def _parse_options(self, input_source):
@@ -1777,9 +1782,9 @@ class DataToolParameter(BaseDataToolParameter):
         # create dictionary and fill default parameters
         d = super(DataToolParameter, self).to_dict(trans)
         extensions = self.extensions
-        datatypes_registery = self._datatypes_registery(trans, self.tool)
-        all_edam_formats = datatypes_registery.edam_formats if hasattr(datatypes_registery, 'edam_formats') else {}
-        all_edam_data = datatypes_registery.edam_data if hasattr(datatypes_registery, 'edam_formats') else {}
+        datatypes_registry = self._datatypes_registry(trans, self.tool)
+        all_edam_formats = datatypes_registry.edam_formats if hasattr(datatypes_registry, 'edam_formats') else {}
+        all_edam_data = datatypes_registry.edam_data if hasattr(datatypes_registry, 'edam_formats') else {}
         edam_formats = [all_edam_formats.get(ext, None) for ext in extensions]
         edam_data = [all_edam_data.get(ext, None) for ext in extensions]
 

From 78e1478c7f9c6f3c232c6c91263c7bb28dec5c88 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 10:31:45 +0100
Subject: [PATCH 21/23] Remove unnecessary tool data table resets

---
 lib/tool_shed/metadata/repository_metadata_manager.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py
index 026ca56d4c4..768eea329ae 100644
--- a/lib/tool_shed/metadata/repository_metadata_manager.py
+++ b/lib/tool_shed/metadata/repository_metadata_manager.py
@@ -809,8 +809,6 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator):
         # Set tool version information for all downloadable changeset revisions.  Get the list of changeset
         # revisions from the changelog.
         self.reset_all_tool_versions(repo)
-        # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
-        self.app.tool_data_tables.data_tables = {}
 
     def reset_all_tool_versions(self, repo):
         """Reset tool version lineage for those changeset revisions that include valid tools."""
@@ -989,7 +987,6 @@ class RepositoryMetadataManager(metadata_generator.MetadataGenerator):
                                                                    self.repository,
                                                                    self.metadata_dict)
             status = 'error'
-        # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
         return message, status
 
     def set_repository_metadata_due_to_new_tip(self, host, content_alert_str=None, **kwd):

From e65a2490c108929043a685eeca6bb89b6ade0b9c Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 10:49:11 +0100
Subject: [PATCH 22/23] Linting fixes

---
 lib/galaxy/tools/repositories.py          | 8 ++++----
 lib/tool_shed/tools/data_table_manager.py | 1 -
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/tools/repositories.py b/lib/galaxy/tools/repositories.py
index 5bc35faef62..dec44e8e336 100644
--- a/lib/galaxy/tools/repositories.py
+++ b/lib/galaxy/tools/repositories.py
@@ -45,10 +45,10 @@ class ValidationContext(object):
                          self.config.tool_data_table_config,
                          self.config.shed_tool_data_table_config}
         for path in cleanup_paths:
-           try:
-               os.remove(path)
-           except Exception:
-               pass
+            try:
+                os.remove(path)
+            except Exception:
+                pass
 
     @staticmethod
     @contextmanager
diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py
index cf5dd21df2f..3f1386efabe 100644
--- a/lib/tool_shed/tools/data_table_manager.py
+++ b/lib/tool_shed/tools/data_table_manager.py
@@ -3,7 +3,6 @@ import os
 import shutil
 from xml.etree import ElementTree as XmlET
 
-from galaxy.tools.data import ToolDataTableManager as tdtm
 from tool_shed.util import hg_util, xml_util
 
 log = logging.getLogger(__name__)

From 4f162f28041d6c0ee3b551d589339d4048b5b7e1 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 3 Dec 2017 15:41:06 +0100
Subject: [PATCH 23/23] Add unit test for ValidationContext and ToolValidator

---
 lib/galaxy/tools/__init__.py                  |  2 +-
 .../bowtie2_indices.loc.sample                | 37 ++++++++++
 test/unit/tools/test_tool_validation.py       | 74 +++++++++++++++++++
 3 files changed, 112 insertions(+), 1 deletion(-)
 create mode 100755 test/shed_functional/test_data/bowtie2_loc_sample/bowtie2_indices.loc.sample
 create mode 100644 test/unit/tools/test_tool_validation.py

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 73aab82bcbe..6cc79b47b36 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1379,7 +1379,7 @@ class Tool(object, Dictifiable):
         for input_param in self.input_params:
             if isinstance(input_param, SelectToolParameter) and input_param.is_dynamic:
                 options = input_param.options
-                if options and options.missing_index_file and input_param not in params:
+                if options and options.tool_data_table and options.tool_data_table.missing_index_file and input_param not in params:
                     params.append(input_param)
         return params
 
diff --git a/test/shed_functional/test_data/bowtie2_loc_sample/bowtie2_indices.loc.sample b/test/shed_functional/test_data/bowtie2_loc_sample/bowtie2_indices.loc.sample
new file mode 100755
index 00000000000..61663caa7c7
--- /dev/null
+++ b/test/shed_functional/test_data/bowtie2_loc_sample/bowtie2_indices.loc.sample
@@ -0,0 +1,37 @@
+# bowtie2_indices.loc.sample
+# This is a *.loc.sample file distributed with Galaxy that enables tools
+# to use a directory of indexed data files. This one is for Bowtie2 and Tophat2.
+# See the wiki: http://wiki.galaxyproject.org/Admin/NGS%20Local%20Setup
+# First create these data files and save them in your own data directory structure.
+# Then, create a bowtie_indices.loc file to use those indexes with tools.
+# Copy this file, save it with the same name (minus the .sample),
+# follow the format examples, and store the result in this directory.
+# The file should include an one line entry for each index set.
+# The path points to the "basename" for the set, not a specific file.
+# It has four text columns seperated by TABS.
+#
+# <unique_build_id>	<dbkey>	<display_name>	<file_base_path>
+#
+# So, for example, if you had hg18 indexes stored in:
+#
+#    /depot/data2/galaxy/hg19/bowtie2/
+#
+# containing hg19 genome and hg19.*.bt2 files, such as:
+#    -rw-rw-r-- 1 james   james   914M Feb 10 18:56 hg19canon.fa
+#    -rw-rw-r-- 1 james   james   914M Feb 10 18:56 hg19canon.1.bt2
+#    -rw-rw-r-- 1 james   james   683M Feb 10 18:56 hg19canon.2.bt2
+#    -rw-rw-r-- 1 james   james   3.3K Feb 10 16:54 hg19canon.3.bt2
+#    -rw-rw-r-- 1 james   james   683M Feb 10 16:54 hg19canon.4.bt2
+#    -rw-rw-r-- 1 james   james   914M Feb 10 20:45 hg19canon.rev.1.bt2
+#    -rw-rw-r-- 1 james   james   683M Feb 10 20:45 hg19canon.rev.2.bt2
+#
+# then the bowtie2_indices.loc entry could look like this:
+#
+#hg19	hg19	Human (hg19)	/depot/data2/galaxy/hg19/bowtie2/hg19canon
+#
+#More examples:
+#
+#mm10	mm10	Mouse (mm10)	/depot/data2/galaxy/mm10/bowtie2/mm10
+#dm3	dm3		D. melanogaster (dm3)	/depot/data2/galaxy/mm10/bowtie2/dm3
+#
+#
diff --git a/test/unit/tools/test_tool_validation.py b/test/unit/tools/test_tool_validation.py
new file mode 100644
index 00000000000..28f5cd211d7
--- /dev/null
+++ b/test/unit/tools/test_tool_validation.py
@@ -0,0 +1,74 @@
+import os
+import shutil
+import tarfile
+import tempfile
+from contextlib import contextmanager
+
+from galaxy.tools.repositories import ValidationContext
+from tool_shed.tools.tool_validator import ToolValidator
+from ..unittest_utils.galaxy_mock import MockApp
+
+CURRENT_DIR = os.path.dirname(__file__)
+BISMARK_TAR = os.path.join(CURRENT_DIR,
+                           '../../shed_functional/test_data/bismark/bismark.tar')
+BOWTIE2_INDICES = os.path.join(CURRENT_DIR,
+                               '../../shed_functional/test_data/bowtie2_loc_sample/bowtie2_indices.loc.sample')
+
+
+def test_validate_valid_tool():
+    with get_tool_validator() as tv, setup_bismark() as repo_dir:
+        full_path = os.path.join(repo_dir, 'bismark_methylation_extractor.xml')
+        tool, valid, message = tv.load_tool_from_config(repository_id=None,
+                                                        full_path=full_path)
+        assert tool.name == 'Bismark'
+        assert not tool.params_with_missing_data_table_entry
+        assert not tool.params_with_missing_index_file
+        assert valid is True
+        assert message is None
+
+
+def test_validate_tool_without_index():
+    with get_tool_validator() as tv, setup_bismark() as repo_dir:
+        full_path = os.path.join(repo_dir, 'bismark_bowtie2_wrapper.xml')
+        tool, valid, message = tv.load_tool_from_config(repository_id=None,
+                                                        full_path=full_path)
+        assert valid is True
+        # tool is missing a data table
+        assert tool.params_with_missing_data_table_entry
+        # check input parameters, which will set up the missing data table (if present in repo_dir)
+        invalid_files_and_errors_tups = tv.check_tool_input_params(repo_dir=os.path.dirname(full_path),
+                                                                   tool_config_name=full_path,
+                                                                   tool=tool,
+                                                                   sample_files=[])
+        # tool is invalid because loc file is not present, but data table is present now
+        assert len(invalid_files_and_errors_tups) == 1
+        assert invalid_files_and_errors_tups[0][1].startswith('This file refers to a file named')
+        assert not tool.params_with_missing_data_table_entry
+        assert tool.params_with_missing_index_file
+        # copy a loc file
+        shutil.copy(BOWTIE2_INDICES, repo_dir)
+        tool, valid, message, sample_files = tv.handle_sample_files_and_load_tool_from_disk(repo_dir, None, full_path, repo_dir)
+        invalid_files_and_errors_tups = tv.check_tool_input_params(repo_dir=repo_dir,
+                                                                   tool_config_name=full_path,
+                                                                   tool=tool,
+                                                                   sample_files=sample_files)
+        # no parameters should be missing
+        assert len(invalid_files_and_errors_tups) == 0
+        assert not tool.params_with_missing_data_table_entry
+        assert not tool.params_with_missing_index_file
+
+
+@contextmanager
+def setup_bismark():
+    repo_dir = tempfile.mkdtemp()
+    with tarfile.open(BISMARK_TAR) as archive:
+        archive.extractall(repo_dir)
+    yield repo_dir
+    shutil.rmtree(repo_dir, ignore_errors=True)
+
+
+@contextmanager
+def get_tool_validator():
+    app = MockApp()
+    with ValidationContext.from_app(app) as validation_context:
+        yield ToolValidator(validation_context)