From c49e7803fa1814abd04afb26b54965fb7af612ac Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 13 Apr 2016 11:12:15 -0400 Subject: [PATCH 01/10] fix biomart url --- tools/data_source/biomart.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/data_source/biomart.xml b/tools/data_source/biomart.xml index db56c299f1d..833a7e7b636 100644 --- a/tools/data_source/biomart.xml +++ b/tools/data_source/biomart.xml @@ -10,7 +10,7 @@ Central server data_source.py $output $__app__.config.output_size_limit - + go to BioMart Central $GALAXY_URL From 7bed69ba17b4ce633c621ee34a7544115eb95970 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 14 Apr 2016 16:56:47 -0400 Subject: [PATCH 02/10] Specify Ensembl biomart server in descriptions. --- tools/data_source/biomart.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/data_source/biomart.xml b/tools/data_source/biomart.xml index 833a7e7b636..989f3537d7e 100644 --- a/tools/data_source/biomart.xml +++ b/tools/data_source/biomart.xml @@ -8,10 +8,10 @@ everything including and beyond the first '&' is truncated from URL. They said they'll let us know when this is fixed at their end. --> - Central server + Ensembl server data_source.py $output $__app__.config.output_size_limit - go to BioMart Central $GALAXY_URL + go to BioMart (Ensembl) $GALAXY_URL From 1e7edf0bd7a003e0abc63c4f0a17fdfac535ade3 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 14 Apr 2016 16:59:25 -0400 Subject: [PATCH 03/10] no parens --- tools/data_source/biomart.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/data_source/biomart.xml b/tools/data_source/biomart.xml index 989f3537d7e..bf4c4c13bd1 100644 --- a/tools/data_source/biomart.xml +++ b/tools/data_source/biomart.xml @@ -11,7 +11,7 @@ Ensembl server data_source.py $output $__app__.config.output_size_limit - go to BioMart (Ensembl) $GALAXY_URL + go to BioMart Ensembl $GALAXY_URL From cc4911019077c5c6e1b4ec293670020f34aed596 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 19 Apr 2016 14:00:50 -0400 Subject: [PATCH 04/10] Enhance to_json such that it can be safely called multiple times --- lib/galaxy/tools/parameters/basic.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 56f9a6d9893..c2883bf2cba 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1673,13 +1673,16 @@ class BaseDataToolParameter( ToolParameter ): def to_json( self, value, app ): def single_to_json( value ): src = None - if isinstance( value, galaxy.model.DatasetCollectionElement ): + if isinstance( value, dict ) and 'src' in value and 'id' in value: + return value + elif isinstance( value, galaxy.model.DatasetCollectionElement ): src = 'dce' elif isinstance( value, app.model.HistoryDatasetCollectionAssociation ): src = 'hdca' - else: + elif hasattr( value, 'id' ): src = 'hda' - return { 'id' : app.security.encode_id( value.id ), 'src' : src } + if src is not None: + return { 'id' : app.security.encode_id( value.id ), 'src' : src } if value not in [ None, '', 'None' ]: if isinstance( value, list ) and len( value ) > 0: From f5060628d2de4802be610fd6caf5c878afdb4312 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 19 Apr 2016 14:19:45 -0400 Subject: [PATCH 05/10] Fix pages encoding. Using str on previously unicodify'd extended set values does not work. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` In [2]: str("Blah blah blah, this breaks the page: ಠ_ಠ") Out[2]: 'Blah blah blah, this breaks the page: \xe0\xb2\xa0_\xe0\xb2\xa0' In [3]: str(unicodify("Blah blah blah, this breaks the page: ಠ_ಠ")) --------------------------------------------------------------------------- UnicodeEncodeError Traceback (most recent call last) in () ----> 1 str(unicodify("Blah blah blah, this breaks the page: ಠ_ಠ")) UnicodeEncodeError: 'ascii' codec can't encode character u'\u0ca0' in position 38: ordinal not in range(128) ``` --- lib/galaxy/util/sanitize_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 2aabbe2dccd..aff64ec34bb 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -194,7 +194,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): def output(self): '''Return processed HTML as a single string''' - return ''.join([str(p) for p in self.pieces]) + return ''.join([unicodify(p) for p in self.pieces]) class _HTMLSanitizer(_BaseHTMLProcessor): From 659a900693b72886fd27731a2568ce71657c4401 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 19 Apr 2016 15:39:51 -0400 Subject: [PATCH 06/10] Simplify expression, trust upstream pieces. --- lib/galaxy/util/sanitize_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index aff64ec34bb..aec55f9f53a 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -194,7 +194,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): def output(self): '''Return processed HTML as a single string''' - return ''.join([unicodify(p) for p in self.pieces]) + return ''.join(self.pieces) class _HTMLSanitizer(_BaseHTMLProcessor): From e13738868519d8c7c6f7dec8cfd598f7929b8828 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 19 Apr 2016 17:32:41 -0400 Subject: [PATCH 07/10] Apply parameter translation for data source tools --- lib/galaxy/tools/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 0fca24488a6..7a90f701fcb 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1099,6 +1099,8 @@ class Tool( object, Dictifiable ): for expanded_incoming in expanded_incomings: params = {} errors = {} + if self.input_translator: + self.input_translator.translate( expanded_incoming ) if not self.check_values: # If `self.check_values` is false we don't do any checking or # processing on input This is used to pass raw values From 7c7d6edbfd94e5b3e187778b89ac1febb5690cb9 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 19 Apr 2016 20:54:24 -0400 Subject: [PATCH 08/10] Swap validate_and_sanitize_basestring to use unicodify --- lib/galaxy/util/validation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py index 7b0686cec19..9bf4a8a5d84 100644 --- a/lib/galaxy/util/validation.py +++ b/lib/galaxy/util/validation.py @@ -3,6 +3,7 @@ TODO: Refactor BaseController references to similar methods to use this module. """ from galaxy import exceptions +from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html from six import string_types, text_type @@ -12,7 +13,7 @@ def validate_and_sanitize_basestring( key, val ): if not isinstance( val, string_types ): raise exceptions.RequestParameterInvalidException( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) - return text_type( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) + return unicodify( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) def validate_and_sanitize_basestring_list( key, val ): From a9280649b201dc8c1ce6e1601c5baaae211acbd7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 20 Apr 2016 09:24:33 -0400 Subject: [PATCH 09/10] Sync job_script module with Pulsar to fix doctest. Following test fails without this change and is fixed by it. ``` nosetests --with-doctest lib/galaxy/jobs/runners/util/job_script ``` I'm not sure why Travis doesn't pick this up - I should figure that out. --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 9f144dc51a4..a08857c2462 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -65,7 +65,7 @@ def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): >>> script.startswith('#!/bin/sh\\n#PBS -test\\n') False >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', headers='#PBS -test') - >>> script.startswith('#!/bin/sh\\n#PBS -test\\n') + >>> script.startswith('#!/bin/bash\\n\\n#PBS -test\\n') True >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', slots_statement='GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"') >>> script.find('GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"\\nexport GALAXY_SLOTS\\n') > 0 @@ -76,7 +76,7 @@ def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): job_instrumenter = kwds.get("job_instrumenter", None) if job_instrumenter: del kwds["job_instrumenter"] - working_directory = kwds["working_directory"] + working_directory = kwds.get("metadata_directory", kwds["working_directory"]) kwds["instrument_pre_commands"] = job_instrumenter.pre_execute_commands(working_directory) or '' kwds["instrument_post_commands"] = job_instrumenter.post_execute_commands(working_directory) or '' From 0cde386a2cfec1c18f68f1dfd939843328f8046f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 20 Apr 2016 10:00:08 -0400 Subject: [PATCH 10/10] Remove more unnecessary unicode/text_type, since sanitize_html will always return unicode. --- lib/galaxy/util/validation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py index 9bf4a8a5d84..a163abf0b44 100644 --- a/lib/galaxy/util/validation.py +++ b/lib/galaxy/util/validation.py @@ -3,23 +3,22 @@ TODO: Refactor BaseController references to similar methods to use this module. """ from galaxy import exceptions -from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html -from six import string_types, text_type +from six import string_types def validate_and_sanitize_basestring( key, val ): if not isinstance( val, string_types ): raise exceptions.RequestParameterInvalidException( '%s must be a string or unicode: %s' % ( key, str( type( val ) ) ) ) - return unicodify( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' ) + return sanitize_html( val, 'utf-8', 'text/html' ) def validate_and_sanitize_basestring_list( key, val ): try: assert isinstance( val, list ) - return [ text_type( sanitize_html( t, 'utf-8', 'text/html' ), 'utf-8' ) for t in val ] + return [ sanitize_html( t, 'utf-8', 'text/html' ) for t in val ] except ( AssertionError, TypeError ): raise exceptions.RequestParameterInvalidException( '%s must be a list of strings: %s' % ( key, str( type( val ) ) ) )