diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index d6c003eea1d..bbb215ed2e2 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1454,7 +1454,8 @@ class JobWrapper(HasResourceParameters): if not os.path.exists(version_filename): version_filename = self.get_version_string_path_legacy() if os.path.exists(version_filename): - self.version_string = open(version_filename).read() + with open(version_filename, 'rb') as fh: + self.version_string = galaxy.util.shrink_and_unicodify(fh.read()) os.unlink(version_filename) outputs_to_working_directory = util.asbool(self.get_destination_configuration("outputs_to_working_directory", False)) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7d0f91185dd..3482203de8c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -289,11 +289,13 @@ class JobLike(object): def set_streams(self, tool_stdout, tool_stderr, job_stdout=None, job_stderr=None, job_messages=None): def shrink_and_unicodify(what, stream): - stream = galaxy.util.unicodify(stream) or u'' - if (len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE): - stream = galaxy.util.shrink_string_by_size(tool_stdout, galaxy.util.DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) - log.info("%s for %s %d is greater than %s, only a portion will be logged to database", what, type(self), self.id, galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) - return stream + if len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE: + log.info("%s for %s %d is greater than %s, only a portion will be logged to database", + what, + type(self), + self.id, + galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) + return galaxy.util.shrink_and_unicodify(stream) self.tool_stdout = shrink_and_unicodify('tool_stdout', tool_stdout) self.tool_stderr = shrink_and_unicodify('tool_stderr', tool_stderr) @@ -2397,6 +2399,14 @@ class DatasetInstance(object): self.parent_id = parent_id self.validation_errors = validation_errors + @property + def peek(self): + return self._peek + + @peek.setter + def peek(self, peek): + self._peek = unicodify(peek, strip_null=True) + def update(self): self.update_time = galaxy.model.orm.now.now() diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e9858e23292..251c2695d17 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -222,7 +222,7 @@ model.HistoryDatasetAssociation.table = Table( Column("name", TrimmedString(255)), Column("info", TrimmedString(255)), Column("blurb", TrimmedString(255)), - Column("peek", TEXT), + Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), Column("metadata", MetadataType(), key="_metadata"), @@ -506,7 +506,7 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("name", TrimmedString(255), index=True), Column("info", TrimmedString(255)), Column("blurb", TrimmedString(255)), - Column("peek", TEXT), + Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), Column("metadata", MetadataType(), key="_metadata"), diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 93c76f36003..686946e9c96 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -402,6 +402,17 @@ def shrink_stream_by_size(value, size, join_by=b"..", left_larger=True, beginnin return unicodify(rval) +def shrink_and_unicodify(stream): + stream = unicodify(stream, strip_null=True) or u'' + if (len(stream) > DATABASE_MAX_STRING_SIZE): + stream = shrink_string_by_size(stream, + DATABASE_MAX_STRING_SIZE, + join_by="\n..\n", + left_larger=True, + beginning_on_size_error=True) + return stream + + def shrink_string_by_size(value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False): if len(value) > size: len_join_by = len(join_by) @@ -993,7 +1004,7 @@ def roundify(amount, sfs=2): return amount[0:sfs] + '0' * (len(amount) - sfs) -def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): +def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False): u""" Returns a Unicode string or None. @@ -1008,7 +1019,7 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): >>> s = u'lâtín strìñg'; assert unicodify(s.encode('latin-1')) == u'l\ufffdt\ufffdn str\ufffd\ufffdg' >>> s = u'lâtín strìñg'; assert unicodify(s.encode('latin-1'), error='ignore') == u'ltn strg' """ - if value is None or isinstance(value, text_type): + if value is None: return value try: if isinstance(value, bytearray): @@ -1025,6 +1036,8 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): msg = "Value '%s' could not be coerced to Unicode" % value log.exception(msg) raise Exception(msg) + if strip_null: + return value.replace('\0', '') return value diff --git a/test/functional/tools/unicode_stream.xml b/test/functional/tools/unicode_stream.xml index 7fb2b00a30d..623b83a61be 100644 --- a/test/functional/tools/unicode_stream.xml +++ b/test/functional/tools/unicode_stream.xml @@ -1,21 +1,26 @@ + echo "\x00" '$out_file1'; +#if $include_null: + echo "\x00" > $out_file1; +#end if +echo '$input1' >> '$out_file1'; cat '$cf'; +echo "\x00"; >&2 cat '$cf'; sh -c "exit $exit" ]]> - ვეპხის ტყაოსანი შოთა რუსთაველი - + ვეპხის ტყაოსანი შოთა რუსთაველი + @@ -34,6 +39,10 @@ sh -c "exit $exit" + + + +