From 562ea1d5e2f5662c7cce0ca954541c3c5833b839 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 27 Nov 2017 09:54:36 +0100 Subject: [PATCH 1/2] Do not write shed_tool_data_table_config in place This should prevent errors such as: ``` galaxy.util ERROR 2017-11-17 09:20:12,842 Error parsing file /data/users/mvandenb/gx/config/shed_tool_data_table_conf.xml Traceback (most recent call last): File "lib/galaxy/util/__init__.py", line 217, in parse_xml root = tree.parse(fname, parser=ElementTree.XMLParser(target=DoctypeSafeCallbackTarget())) File "/usr/lib/python2.7/xml/etree/ElementTree.py", line 657, in parse self._root = parser.close() File "/usr/lib/python2.7/xml/etree/ElementTree.py", line 1654, in close self._raiseerror(v) File "/usr/lib/python2.7/xml/etree/ElementTree.py", line 1506, in _raiseerror raise err ParseError: no element found: line 1, column 0 ``` reported in https://github.com/galaxyproject/galaxy/issues/5031. --- lib/galaxy/tools/data/__init__.py | 3 +- lib/galaxy/util/renamed_temporary_file.py | 42 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/util/renamed_temporary_file.py diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index d3ec66072b7..95ea5c0470d 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -21,6 +21,7 @@ import requests from galaxy import util from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict +from galaxy.util.renamed_temporary_file import RenamedTemporaryFile log = logging.getLogger(__name__) @@ -186,7 +187,7 @@ class ToolDataTableManager(object): # add new elems out_elems.extend(new_elems) out_path_is_new = not os.path.exists(full_path) - with open(full_path, 'wb') as out: + with RenamedTemporaryFile(full_path) as out: out.write('\n\n') for elem in out_elems: out.write(util.xml_to_string(elem, pretty=True)) diff --git a/lib/galaxy/util/renamed_temporary_file.py b/lib/galaxy/util/renamed_temporary_file.py new file mode 100644 index 00000000000..ecbfb7a38c3 --- /dev/null +++ b/lib/galaxy/util/renamed_temporary_file.py @@ -0,0 +1,42 @@ +"""Safely write file to temporary file and then move file into place.""" +# Copied from https://stackoverflow.com/a/12007885. +import os +import tempfile + + +class RenamedTemporaryFile(object): + """ + A temporary file object which will be renamed to the specified + path on exit. + """ + def __init__(self, final_path, **kwargs): + tmpfile_dir = kwargs.pop('dir', None) + + # Put temporary file in the same directory as the location for the + # final file so that an atomic move into place can occur. + + if tmpfile_dir is None: + tmpfile_dir = os.path.dirname(final_path) + + self.tmpfile = tempfile.NamedTemporaryFile(dir=tmpfile_dir, **kwargs) + self.final_path = final_path + + def __getattr__(self, attr): + """ + Delegate attribute access to the underlying temporary file object. + """ + return getattr(self.tmpfile, attr) + + def __enter__(self): + self.tmpfile.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is None: + self.tmpfile.delete = False + result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb) + os.rename(self.tmpfile.name, self.final_path) + else: + result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb) + + return result From eb3d2eac203ccc6673a961b842117176535a8b24 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 27 Nov 2017 11:25:37 +0100 Subject: [PATCH 2/2] Fix installation of data manager revision updates Without this updates of data managers fails with ``` AttributeError: 'NoneType' object has no attribute 'get' File "galaxy/web/framework/middleware/sentry.py", line 40, in __call__ iterable = self.application(environ, start_response) File "/bioinfo/guests/mvandenb/galaxy/.venv/local/lib/python2.7/site-packages/paste/recursive.py", line 85, in __call__ return self.application(environ, start_response) File "/bioinfo/guests/mvandenb/galaxy/.venv/local/lib/python2.7/site-packages/paste/httpexceptions.py", line 640, in __call__ return self.application(environ, start_response) File "galaxy/web/framework/base.py", line 136, in __call__ return self.handle_request(environ, start_response) File "galaxy/web/framework/base.py", line 215, in handle_request body = method(trans, **kwargs) File "galaxy/web/framework/decorators.py", line 98, in decorator return func(self, trans, *args, **kwargs) File "galaxy/webapps/galaxy/controllers/admin_toolshed.py", line 1849, in update_to_changeset_revision persist=True) File "tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py", line 30, in __init__ metadata_dict=metadata_dict, user=None) File "tool_shed/metadata/metadata_generator.py", line 63, in __init__ self.metadata_dict = {'shed_config_filename': self.shed_config_dict.get('config_filename', None)} ``` --- lib/galaxy/model/tool_shed_install/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index f36ae712f10..a8cd0a1106e 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -143,7 +143,7 @@ class ToolShedRepository(object): if tool_id in tool_ids: self.shed_config_filename = name return shed_tool_conf_dict - if self.includes_datatypes: + if self.includes_datatypes or self.includes_data_managers: # We need to search by file paths here, which is less desirable. tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url(self.tool_shed) for shed_tool_conf_dict in app.toolbox.dynamic_confs(include_migrated_tool_conf=True):