From f38ff2e0b1067ca1902d7a505af5aa4700996c03 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 3 May 2019 16:18:21 +0200 Subject: [PATCH 1/4] Change unicode test tool To produce NULL character. This tool fails locally but not if testing against the jenkins framework tests or using external docker-galaxy-stable. I don't understand why. --- test/functional/tools/unicode_stream.xml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/unicode_stream.xml b/test/functional/tools/unicode_stream.xml index 7fb2b00a30d..623b83a61be 100644 --- a/test/functional/tools/unicode_stream.xml +++ b/test/functional/tools/unicode_stream.xml @@ -1,21 +1,26 @@ + echo "\x00" '$out_file1'; +#if $include_null: + echo "\x00" > $out_file1; +#end if +echo '$input1' >> '$out_file1'; cat '$cf'; +echo "\x00"; >&2 cat '$cf'; sh -c "exit $exit" ]]> - ვეპხის ტყაოსანი შოთა რუსთაველი - + ვეპხის ტყაოსანი შოთა რუსთაველი + @@ -34,6 +39,10 @@ sh -c "exit $exit" + + + + From a8f050f65df15cafb99e72375ad111e6baf7df00 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 15 Jun 2019 16:52:24 +0200 Subject: [PATCH 2/4] Strip unicode null from stdout/stderr/tool_version --- lib/galaxy/jobs/__init__.py | 3 ++- lib/galaxy/model/__init__.py | 12 +++++++----- lib/galaxy/util/__init__.py | 21 +++++++++++++++++++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 00f97cf3c39..101c0cda08d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1286,7 +1286,8 @@ class JobWrapper(HasResourceParameters): if not os.path.exists(version_filename): version_filename = self.get_version_string_path_legacy() if os.path.exists(version_filename): - self.version_string = open(version_filename).read() + with open(version_filename, 'rb') as fh: + self.version_string = galaxy.util.shrink_and_unicodify(fh.read()) os.unlink(version_filename) outputs_to_working_directory = util.asbool(self.get_destination_configuration("outputs_to_working_directory", False)) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1448ce6f75e..99bef225138 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -289,11 +289,13 @@ class JobLike(object): def set_streams(self, tool_stdout, tool_stderr, job_stdout=None, job_stderr=None, job_messages=None): def shrink_and_unicodify(what, stream): - stream = galaxy.util.unicodify(stream) or u'' - if (len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE): - stream = galaxy.util.shrink_string_by_size(tool_stdout, galaxy.util.DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) - log.info("%s for %s %d is greater than %s, only a portion will be logged to database", what, type(self), self.id, galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) - return stream + if len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE: + log.info("%s for %s %d is greater than %s, only a portion will be logged to database", + what, + type(self), + self.id, + galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) + return galaxy.util.shrink_and_unicodify(stream) self.tool_stdout = shrink_and_unicodify('tool_stdout', tool_stdout) self.tool_stderr = shrink_and_unicodify('tool_stderr', tool_stderr) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 93c76f36003..ff80f1df795 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -402,6 +402,17 @@ def shrink_stream_by_size(value, size, join_by=b"..", left_larger=True, beginnin return unicodify(rval) +def shrink_and_unicodify(stream): + stream = unicodify(stream, strip_null=True) or u'' + if (len(stream) > DATABASE_MAX_STRING_SIZE): + stream = shrink_string_by_size(stream, + DATABASE_MAX_STRING_SIZE, + join_by="\n..\n", + left_larger=True, + beginning_on_size_error=True) + return stream + + def shrink_string_by_size(value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False): if len(value) > size: len_join_by = len(join_by) @@ -993,7 +1004,7 @@ def roundify(amount, sfs=2): return amount[0:sfs] + '0' * (len(amount) - sfs) -def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): +def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False): u""" Returns a Unicode string or None. @@ -1008,7 +1019,11 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): >>> s = u'lâtín strìñg'; assert unicodify(s.encode('latin-1')) == u'l\ufffdt\ufffdn str\ufffd\ufffdg' >>> s = u'lâtín strìñg'; assert unicodify(s.encode('latin-1'), error='ignore') == u'ltn strg' """ - if value is None or isinstance(value, text_type): + if value is None: + return value + if isinstance(value, text_type): + if strip_null: + return value.replace('\0', '') return value try: if isinstance(value, bytearray): @@ -1025,6 +1040,8 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): msg = "Value '%s' could not be coerced to Unicode" % value log.exception(msg) raise Exception(msg) + if strip_null: + return value.replace('\0', '') return value From af9c924f62a840b206b2e95de28182d6765310ec Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 15 Jun 2019 16:52:41 +0200 Subject: [PATCH 3/4] Strip unicode null from peek --- lib/galaxy/model/__init__.py | 8 ++++++++ lib/galaxy/model/mapping.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 99bef225138..95efaa76ecc 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2402,6 +2402,14 @@ class DatasetInstance(object): self.parent_id = parent_id self.validation_errors = validation_errors + @property + def peek(self): + return self._peek + + @peek.setter + def peek(self, peek): + self._peek = unicodify(peek, strip_null=True) + def update(self): self.update_time = galaxy.model.orm.now.now() diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f7f563c54c8..617e690bd02 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -222,7 +222,7 @@ model.HistoryDatasetAssociation.table = Table( Column("name", TrimmedString(255)), Column("info", TrimmedString(255)), Column("blurb", TrimmedString(255)), - Column("peek", TEXT), + Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), Column("metadata", MetadataType(), key="_metadata"), @@ -505,7 +505,7 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("name", TrimmedString(255), index=True), Column("info", TrimmedString(255)), Column("blurb", TrimmedString(255)), - Column("peek", TEXT), + Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), Column("metadata", MetadataType(), key="_metadata"), From f0f02c7a8c6a340a60c473f116043b05eadb4f88 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 16 Jun 2019 22:24:28 +0200 Subject: [PATCH 4/4] Simplify unicodify Thanks Nicola! --- lib/galaxy/util/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index ff80f1df795..686946e9c96 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1021,10 +1021,6 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals """ if value is None: return value - if isinstance(value, text_type): - if strip_null: - return value.replace('\0', '') - return value try: if isinstance(value, bytearray): value = bytes(value)