From ace30535b29451d6d4b3f080c6a57733ae7d09ac Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 13 Dec 2014 22:09:51 -0500 Subject: [PATCH] Improvements to loading help text from tools. Do not convert rst to a mako template until needed, this is a costly operation and has the potential to speed update Galaxy start time. Move logic related to parsing of simple help text blocks out of tool and into the new tool parser interface and add implementation for YAML-based tools as well as unit tests for both. More advanced, multi-page tool help is still possible, workflows with the old tool form, but is only available to XML-based tools. --- lib/galaxy/tools/__init__.py | 103 +++++++++++++++++---------- lib/galaxy/tools/parser/interface.py | 6 ++ lib/galaxy/tools/parser/xml.py | 4 ++ lib/galaxy/tools/parser/yaml.py | 3 + test/unit/tools/test_parsing.py | 12 ++++ 5 files changed, 91 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e7d99183efa..59d2ab1d95b 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -101,6 +101,8 @@ JOB_RESOURCE_CONDITIONAL_XML = """ """ +HELP_UNINITIALIZED = threading.Lock() + class ToolNotFoundException( Exception ): pass @@ -1680,43 +1682,9 @@ class Tool( object, Dictifiable ): This implementation supports multiple pages. """ # TODO: Allow raw HTML or an external link. - self.help = None - self.help_by_page = list() - help_header = "" - help_footer = "" - if hasattr( tool_source, 'root' ) and tool_source.root.find( 'help' ) is not None: - self.help = tool_source.root.find( 'help' ) - if self.repository_id and self.help.text.find( '.. image:: ' ) >= 0: - # Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy. - try: - self.help.text = suc.set_image_paths( self.app, self.repository_id, self.help.text ) - except Exception, e: - log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) ) - help_pages = self.help.findall( "page" ) - help_header = self.help.text - try: - self.help = Template( rst_to_html(self.help.text), input_encoding='utf-8', - output_encoding='utf-8', default_filters=[ 'decode.utf8' ], - encoding_errors='replace' ) - except: - log.exception( "error in help for tool %s" % self.name ) - # Multiple help page case - if help_pages: - for help_page in help_pages: - self.help_by_page.append( help_page.text ) - help_footer = help_footer + help_page.tail - # Each page has to rendered all-together because of backreferences allowed by rst - try: - self.help_by_page = [ Template( rst_to_html( help_header + x + help_footer ), - input_encoding='utf-8', output_encoding='utf-8', - default_filters=[ 'decode.utf8' ], - encoding_errors='replace' ) - for x in self.help_by_page ] - except: - log.exception( "error in multi-page help for tool %s" % self.name ) - # Pad out help pages to match npages ... could this be done better? - while len( self.help_by_page ) < self.npages: - self.help_by_page.append( self.help ) + self.__help = HELP_UNINITIALIZED + self.__help_by_page = HELP_UNINITIALIZED + self.__help_source = tool_source def parse_outputs( self, tool_source ): """ @@ -1891,6 +1859,67 @@ class Tool( object, Dictifiable ): self.repository_owner = tool_shed_repository.owner self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + @property + def help(self): + if self.__help is HELP_UNINITIALIZED: + self.__ensure_help() + return self.__help + + @property + def help_by_page(self): + if self.__help_by_page is HELP_UNINITIALIZED: + self.__ensure_help() + return self.__help_by_page + + def __ensure_help(self): + with HELP_UNINITIALIZED: + if self.__help is HELP_UNINITIALIZED: + self.__inititalize_help() + + def __inititalize_help(self): + tool_source = self.__help_source + self.__help = None + self.__help_by_page = [] + help_header = "" + help_footer = "" + help_text = tool_source.parse_help() + if help_text is not None: + if self.repository_id and help_text.find( '.. image:: ' ) >= 0: + # Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy. + try: + help_text = suc.set_image_paths( self.app, self.repository_id, help_text ) + except Exception, e: + log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) ) + try: + self.__help = Template( rst_to_html(help_text), input_encoding='utf-8', + output_encoding='utf-8', default_filters=[ 'decode.utf8' ], + encoding_errors='replace' ) + except: + log.exception( "error in help for tool %s" % self.name ) + + # Handle deprecated multi-page help text in XML case. + if hasattr(tool_source, "root"): + help_elem = tool_source.root.find("help") + help_header = help_text + help_pages = help_elem.findall( "page" ) + # Multiple help page case + if help_pages: + for help_page in help_pages: + self.__help_by_page.append( help_page.text ) + help_footer = help_footer + help_page.tail + # Each page has to rendered all-together because of backreferences allowed by rst + try: + self.__help_by_page = [ Template( rst_to_html( help_header + x + help_footer ), + input_encoding='utf-8', output_encoding='utf-8', + default_filters=[ 'decode.utf8' ], + encoding_errors='replace' ) + for x in self.__help_by_page ] + except: + log.exception( "error in multi-page help for tool %s" % self.name ) + # Pad out help pages to match npages ... could this be done better? + while len( self.__help_by_page ) < self.npages: + self.__help_by_page.append( self.__help ) + def check_workflow_compatible( self, tool_source ): """ Determine if a tool can be used in workflows. External tools and the diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py index c9e858ff463..494e03ab7c5 100644 --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -140,6 +140,12 @@ class ToolSource(object): """ return [], [] + @abstractmethod + def parse_help(self): + """ Return RST definition of help text for tool or None if the tool + doesn't define help text. + """ + def parse_tests_to_dict(self): return {'tests': []} diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index f6354ea3444..b394b25dc84 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -162,6 +162,10 @@ class XmlToolSource(ToolSource): parser = StdioParser(self.root) return parser.stdio_exit_codes, parser.stdio_regexes + def parse_help(self): + help_elem = self.root.find( 'help' ) + return help_elem.text if help_elem is not None else None + def parse_tests_to_dict(self): tests_elem = self.root.find("tests") tests = [] diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 5b0b993b7cb..3bd7519d3d3 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -68,6 +68,9 @@ class YamlToolSource(ToolSource): exit_code_lower.error_level = StdioErrorLevel.FATAL return [exit_code_lower, exit_code_high], [] + def parse_help(self): + return self.root_dict.get("help", None) + def parse_outputs(self, tool): outputs = self.root_dict.get("outputs", {}) output_defs = [] diff --git a/test/unit/tools/test_parsing.py b/test/unit/tools/test_parsing.py index 02a36322cea..7fd1102339d 100644 --- a/test/unit/tools/test_parsing.py +++ b/test/unit/tools/test_parsing.py @@ -26,6 +26,7 @@ TOOL_XML_1 = """ + This is HELP TEXT1!!! @@ -75,6 +76,9 @@ inputs: type: data - name: nestsample type: text +help: +| + This is HELP TEXT2!!! tests: - inputs: foo: 5 @@ -173,6 +177,10 @@ class XmlLoaderTestCase(BaseLoaderTestCase): assert exit[0].range_start == 1 assert isinf(exit[0].range_end) + def test_help(self): + help_text = self._tool_source.parse_help() + assert help_text.strip() == "This is HELP TEXT1!!!" + def test_tests(self): tests_dict = self._tool_source.parse_tests_to_dict() tests = tests_dict["tests"] @@ -278,6 +286,10 @@ class YamlLoaderTestCase(BaseLoaderTestCase): assert exit[1].range_start == 1 assert isinf(exit[1].range_end) + def test_help(self): + help_text = self._tool_source.parse_help() + assert help_text.strip() == "This is HELP TEXT2!!!" + def test_inputs(self): input_pages = self._tool_source.parse_input_pages() assert input_pages.inputs_defined