From 36d9514e26037a927ad8becab9a834a4807abc50 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 19 Dec 2015 11:51:08 +0000 Subject: [PATCH] More Python 3 fixes. This fixes all unit tests for code exclusively covering the stuff I'm moving into the galaxy-lib that will be used by planemo, pulsar, and future tooling efforts. --- lib/galaxy/objectstore/pulsar.py | 8 +++++++- lib/galaxy/tools/parser/output_objects.py | 2 +- lib/galaxy/tools/toolbox/base.py | 4 ++-- lib/galaxy/tools/toolbox/panel.py | 6 +++--- lib/galaxy/util/__init__.py | 15 ++++++++++++--- lib/galaxy/util/expressions.py | 21 +++++++++++++++------ lib/galaxy/util/object_wrapper.py | 2 +- lib/galaxy/util/topsort.py | 11 ++++++++--- test/unit/test_objectstore.py | 17 +++++++---------- 9 files changed, 56 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/objectstore/pulsar.py b/lib/galaxy/objectstore/pulsar.py index 08f51ea14dd..f4be1748bb2 100644 --- a/lib/galaxy/objectstore/pulsar.py +++ b/lib/galaxy/objectstore/pulsar.py @@ -1,6 +1,10 @@ from __future__ import absolute_import # Need to import pulsar_client absolutely. from ..objectstore import ObjectStore -from pulsar.client.manager import ObjectStoreClientManager + +try: + from pulsar.client.manager import ObjectStoreClientManager +except ImportError: + ObjectStoreClientManager = None class PulsarObjectStore(ObjectStore): @@ -61,6 +65,8 @@ class PulsarObjectStore(ObjectStore): pass def __build_pulsar_client(self, config_xml): + if ObjectStoreClientManager is None: + raise Exception("Pulsar client code not available, cannot use this module.") url = config_xml.get("url") private_token = config_xml.get("private_token", None) transport = config_xml.get("transport", None) diff --git a/lib/galaxy/tools/parser/output_objects.py b/lib/galaxy/tools/parser/output_objects.py index 5ded72bbcff..8ea45f081cf 100644 --- a/lib/galaxy/tools/parser/output_objects.py +++ b/lib/galaxy/tools/parser/output_objects.py @@ -2,7 +2,7 @@ from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict -class ToolOutputBase( object, Dictifiable ): +class ToolOutputBase( Dictifiable, object ): def __init__( self, name, label=None, filters=None, hidden=False ): super( ToolOutputBase, self ).__init__() diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 8831e7f7309..3ab219be30f 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -7,7 +7,7 @@ import tempfile from sqlalchemy import and_ from markupsafe import escape -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from galaxy.util.dictifiable import Dictifiable @@ -36,7 +36,7 @@ from galaxy.web.form_builder import SelectField log = logging.getLogger( __name__ ) -class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): +class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ Abstract container for managing a ToolPanel - containing tools and workflows optionally in labelled sections. diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py index 7d0e85f7c4f..1f6ec7efbbb 100644 --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -40,7 +40,7 @@ class HasPanelItems: yield (panel_key, panel_type, panel_value) -class ToolSection( object, Dictifiable, HasPanelItems ): +class ToolSection( Dictifiable, HasPanelItems, object ): """ A group of tools with similar type/purpose that will be displayed as a group in the user interface. @@ -84,7 +84,7 @@ class ToolSection( object, Dictifiable, HasPanelItems ): return self.elems -class ToolSectionLabel( object, Dictifiable ): +class ToolSectionLabel( Dictifiable, object ): """ A label for a set of tools that can be displayed above groups of tools and sections in the user interface @@ -104,7 +104,7 @@ class ToolSectionLabel( object, Dictifiable ): return super( ToolSectionLabel, self ).to_dict() -class ToolPanelElements( odict, HasPanelItems ): +class ToolPanelElements( HasPanelItems, odict ): """ Represents an ordered dictionary of tool entries - abstraction used both by tool panel itself (normal and integrated) and its sections. """ diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 3335bb1b23c..7cfbfc263cd 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -27,6 +27,7 @@ from six.moves.urllib import parse as urlparse from galaxy.util import json from datetime import datetime +from six import PY3 from six import string_types, text_type from six.moves import xrange from six.moves import email_mime_text @@ -43,6 +44,13 @@ from xml.etree import ElementTree, ElementInclude from .inflection import Inflector, English inflector = Inflector(English) +if PY3: + def list_map(f, input): + return list(map(f, input)) +else: + list_map = map + + log = logging.getLogger(__name__) _lock = threading.RLock() @@ -145,7 +153,8 @@ def unique_id(KEY_SIZE=128): >>> len(set(ids)) 1000 """ - return md5(str( random.getrandbits( KEY_SIZE ) )).hexdigest() + random_bits = text_type(random.getrandbits(KEY_SIZE)).encode("UTF-8") + return md5(random_bits).hexdigest() def parse_xml( fname ): @@ -418,7 +427,7 @@ def sanitize_text( text, valid_characters=valid_chars, character_map=mapped_char and lists of strings; non-string entities will be cast to strings. """ if isinstance( text, list ): - return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) + return list_map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), text ) if not isinstance( text, string_types ): text = smart_str( text ) return _sanitize_text_helper( text, valid_characters=valid_characters, character_map=character_map ) @@ -457,7 +466,7 @@ def sanitize_param( value, valid_characters=valid_chars, character_map=mapped_ch if isinstance( value, string_types ): return sanitize_text( value, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ) elif isinstance( value, list ): - return map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) + return list_map( lambda x: sanitize_text( x, valid_characters=valid_characters, character_map=character_map, invalid_character=invalid_character ), value ) else: raise Exception('Unknown parameter type (%s)' % ( type( value ) )) diff --git a/lib/galaxy/util/expressions.py b/lib/galaxy/util/expressions.py index c57b4ecab0f..2b106a87fd5 100644 --- a/lib/galaxy/util/expressions.py +++ b/lib/galaxy/util/expressions.py @@ -5,14 +5,11 @@ For the moment this depends on python's eval. In the future it should be replaced with a "safe" parser. """ -try: - from UserDict import DictMixin as AbstractMapping -except ImportError: - # PY3 - from collections import MutableMapping as AbstractMapping +from collections import MutableMapping +from itertools import chain -class ExpressionContext( object, AbstractMapping ): +class ExpressionContext( MutableMapping ): def __init__( self, dict, parent=None ): """ Create a new expression context that looks for values in the @@ -21,6 +18,18 @@ class ExpressionContext( object, AbstractMapping ): self.dict = dict self.parent = parent + def __delitem__(self, key): + if key in self.dict: + del self.dict[key] + elif self.parent is not None and key in self.parent: + del self.parent[key] + + def __iter__(self): + return chain(iter(self.dict), iter(self.parent or [])) + + def __len__(self): + return len(self.dict) + len(self.parent or []) + def __getitem__( self, key ): if key in self.dict: return self.dict[key] diff --git a/lib/galaxy/util/object_wrapper.py b/lib/galaxy/util/object_wrapper.py index d28146a651f..24214270bd0 100644 --- a/lib/galaxy/util/object_wrapper.py +++ b/lib/galaxy/util/object_wrapper.py @@ -74,7 +74,7 @@ __WRAP_MAPPINGS__ = ( dict, UserDict, ) # Define the set of characters that are not sanitized, and define a set of mappings for those that are. # characters that are valid -VALID_CHARACTERS = set( string.letters + string.digits + " -=_.()/+*^,:?!@" ) +VALID_CHARACTERS = set( string.ascii_letters + string.digits + " -=_.()/+*^,:?!@" ) # characters that are allowed but need to be escaped CHARACTER_MAP = { '>': '__gt__', diff --git a/lib/galaxy/util/topsort.py b/lib/galaxy/util/topsort.py index b9ea85ec7c5..a2666a2fa42 100644 --- a/lib/galaxy/util/topsort.py +++ b/lib/galaxy/util/topsort.py @@ -37,7 +37,12 @@ many methods to help analyze and break the cycles. This requires a good deal more code than topsort itself! """ from galaxy.util.odict import odict as OrderedDict -from exceptions import Exception +from six import PY3 +if PY3: + def list_filter(f, lst): + return list(filter(f, lst)) +else: + list_filter = filter class CycleError(Exception): @@ -159,8 +164,8 @@ def topsort(pairlist): numpreds, successors = _numpreds_and_successors_from_pairlist(pairlist) # suck up everything without a predecessor - answer = filter(lambda x, numpreds=numpreds: numpreds[x] == 0, - numpreds.keys()) + answer = list_filter(lambda x, numpreds=numpreds: numpreds[x] == 0, + numpreds.keys()) # for everything in answer, knock down the pred count on # its successors; note that answer grows *in* the loop diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index 3f0b96ff52f..5812ecfed28 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -2,10 +2,7 @@ import os from shutil import rmtree from string import Template from tempfile import mkdtemp -try: - from galaxy import objectstore -except ImportError: - from lwr import objectstore +from galaxy import objectstore from contextlib import contextmanager DISK_TEST_CONFIG = """ @@ -26,22 +23,22 @@ def test_disk_store(): # Write empty dataset 2 in second backend, ensure it is empty and # exists. empty_dataset = MockDataset(2) - directory.write(b"", "files1/000/dataset_2.dat") + directory.write("", "files1/000/dataset_2.dat") assert object_store.exists(empty_dataset) assert object_store.empty(empty_dataset) # Write non-empty dataset in backend 1, test it is not emtpy & exists. hello_world_dataset = MockDataset(3) - directory.write(b"Hello World!", "files1/000/dataset_3.dat") + directory.write("Hello World!", "files1/000/dataset_3.dat") assert object_store.exists(hello_world_dataset) assert not object_store.empty(hello_world_dataset) # Test get_data data = object_store.get_data(hello_world_dataset) - assert data == b"Hello World!" + assert data == "Hello World!" data = object_store.get_data(hello_world_dataset, start=1, count=6) - assert data == b"ello W" + assert data == "ello W" # Test Size @@ -60,13 +57,13 @@ def test_disk_store(): output_dataset = MockDataset(4) output_real_path = os.path.join(directory.temp_directory, "files1", "000", "dataset_4.dat") assert not os.path.exists(output_real_path) - output_working_path = directory.write(b"NEW CONTENTS", "job_working_directory1/example_output") + output_working_path = directory.write("NEW CONTENTS", "job_working_directory1/example_output") object_store.update_from_file(output_dataset, file_name=output_working_path, create=True) assert os.path.exists(output_real_path) # Test delete to_delete_dataset = MockDataset(5) - to_delete_real_path = directory.write(b"content to be deleted!", "files1/000/dataset_5.dat") + to_delete_real_path = directory.write("content to be deleted!", "files1/000/dataset_5.dat") assert object_store.exists(to_delete_dataset) assert object_store.delete(to_delete_dataset) assert not object_store.exists(to_delete_dataset)