From 633061f3e0e95041383901378e33f8b41b83717e Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sat, 4 Jul 2015 15:18:50 -0400 Subject: [PATCH 01/52] WIP: Stop using eggs, start using prebuilt wheels. --- lib/galaxy/__init__.py | 79 +- lib/galaxy/eggs/__init__.py | 526 +--- lib/galaxy/eggs/dist.py | 92 - lib/galaxy/eggs/scramble.py | 248 -- lib/galaxy/wheels.py | 67 + lib/pkg_resources.py | 2625 ----------------- run.sh | 16 +- scripts/check_eggs.py | 51 - scripts/common_startup.sh | 63 +- scripts/dist-scramble.py | 91 - scripts/fetch_eggs.py | 78 - scripts/make_egg_packager.py | 71 - scripts/scramble.py | 58 - scripts/scramble/lib/scramble_lib.py | 158 - .../scramble/patches/Cheetah/SetupConfig.py | 103 - .../scramble/patches/MySQL_python/site.cfg | 18 - scripts/scramble/patches/PasteScript/setup.py | 145 - .../scramble/patches/python_daemon/setup.py | 64 - .../scramble/scripts/DRMAA_python-macosx.py | 49 - .../scramble/scripts/DRMAA_python-solaris.py | 61 - scripts/scramble/scripts/DRMAA_python.py | 41 - .../scramble/scripts/MySQL_python-solaris.py | 81 - scripts/scramble/scripts/MySQL_python.py | 78 - scripts/scramble/scripts/PyRods.py | 33 - scripts/scramble/scripts/bx_python-solaris.py | 33 - scripts/scramble/scripts/generic.py | 29 - scripts/scramble/scripts/pbs_python.py | 49 - scripts/scramble/scripts/psycopg2-linux.py | 80 - scripts/scramble/scripts/psycopg2-macosx.py | 85 - scripts/scramble/scripts/psycopg2-solaris.py | 82 - scripts/scramble/scripts/psycopg2.py | 79 - scripts/scramble/scripts/pysam-solaris.py | 30 - scripts/scramble/scripts/pysqlite.py | 36 - scripts/test_dist_egg.py | 53 - 34 files changed, 129 insertions(+), 5323 deletions(-) delete mode 100644 lib/galaxy/eggs/dist.py delete mode 100644 lib/galaxy/eggs/scramble.py create mode 100644 lib/galaxy/wheels.py delete mode 100644 lib/pkg_resources.py delete mode 100644 scripts/check_eggs.py delete mode 100644 scripts/dist-scramble.py delete mode 100755 scripts/fetch_eggs.py delete mode 100644 scripts/make_egg_packager.py delete mode 100644 scripts/scramble.py delete mode 100644 scripts/scramble/lib/scramble_lib.py delete mode 100644 scripts/scramble/patches/Cheetah/SetupConfig.py delete mode 100644 scripts/scramble/patches/MySQL_python/site.cfg delete mode 100644 scripts/scramble/patches/PasteScript/setup.py delete mode 100644 scripts/scramble/patches/python_daemon/setup.py delete mode 100644 scripts/scramble/scripts/DRMAA_python-macosx.py delete mode 100644 scripts/scramble/scripts/DRMAA_python-solaris.py delete mode 100644 scripts/scramble/scripts/DRMAA_python.py delete mode 100644 scripts/scramble/scripts/MySQL_python-solaris.py delete mode 100644 scripts/scramble/scripts/MySQL_python.py delete mode 100644 scripts/scramble/scripts/PyRods.py delete mode 100644 scripts/scramble/scripts/bx_python-solaris.py delete mode 100644 scripts/scramble/scripts/generic.py delete mode 100644 scripts/scramble/scripts/pbs_python.py delete mode 100644 scripts/scramble/scripts/psycopg2-linux.py delete mode 100644 scripts/scramble/scripts/psycopg2-macosx.py delete mode 100644 scripts/scramble/scripts/psycopg2-solaris.py delete mode 100644 scripts/scramble/scripts/psycopg2.py delete mode 100644 scripts/scramble/scripts/pysam-solaris.py delete mode 100644 scripts/scramble/scripts/pysqlite.py delete mode 100644 scripts/test_dist_egg.py diff --git a/lib/galaxy/__init__.py b/lib/galaxy/__init__.py index 38baf123d8b..07a3f6c52a0 100644 --- a/lib/galaxy/__init__.py +++ b/lib/galaxy/__init__.py @@ -2,82 +2,9 @@ Galaxy root package -- this is a namespace package. """ -__import__( "pkg_resources" ).declare_namespace( __name__ ) +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) -import re -import os -import sys -import platform - -import pkg_resources - -# patch get_platform() for better ABI recognition -def _get_build_platform(): - plat = pkg_resources._get_build_platform() - if sys.platform == 'darwin': - # Assert OS X version is new enough - current_minor_ver = int( platform.mac_ver()[0].split( '.' )[1] ) - assert current_minor_ver >= 6, 'Galaxy is not compatible with Mac OS X < 10.6 (your version is %s)' % platform.mac_ver()[0] - # Python build target may be even older, but this will prevent it from - # fetching our 10.6+ eggs, so fix if necessary (newer versions will be - # fine) - plat_split = plat.split( '-' ) - build_minor_ver = int( plat_split[1].split( '.' )[1] ) - if build_minor_ver < 6: - plat_split[1] = '10.6' - # Our intel (dual arch) eggs will work fine on single-arch builds - if plat_split[-1] in ( 'i386', 'x86_64' ): - plat_split[-1] = 'intel' - # Ditto universal (if you're not on PPC) - if plat_split[-1] == 'universal' and platform.processor() != 'powerpc': - plat_split[-1] = 'intel' - plat = '-'.join( plat_split ) - elif sys.platform == "linux2" and sys.maxint < 2**31 and plat.endswith( '-x86_64' ): - # 32 bit Python on 64 bit Linux - plat = plat.replace( '-x86_64', '-i686' ) - if not (plat.endswith('-ucs2') or plat.endswith('-ucs4')): - if sys.maxunicode > 2**16: - plat += '-ucs4' - else: - plat += '-ucs2' - return plat -try: - assert pkg_resources._get_build_platform -except: - pkg_resources._get_build_platform = pkg_resources.get_build_platform - pkg_resources.get_build_platform = _get_build_platform - pkg_resources.get_platform = _get_build_platform - -# patch to insert eggs at the beginning of sys.path instead of at the end -def _insert_on(self, path, loc = None): - """Insert self.location in path before its nearest parent directory""" - - loc = loc or self.location - if not loc: - return - - nloc = pkg_resources._normalize_cached(loc) - npath= [(p and pkg_resources._normalize_cached(p) or p) for p in path] - - if path is sys.path: - self.check_version_conflict() - path.insert(0, loc) - - # remove dups - while 1: - try: - np = npath.index(nloc, 1) - except ValueError: - break - else: - del npath[np], path[np] - - return -try: - assert pkg_resources.Distribution._insert_on -except: - pkg_resources.Distribution._insert_on = pkg_resources.Distribution.insert_on - pkg_resources.Distribution.insert_on = _insert_on # compat: BadZipFile introduced in Python 2.7 import zipfile @@ -91,5 +18,3 @@ if not hasattr( logging, 'NullHandler' ): def emit( self, record ): pass logging.NullHandler = NullHandler - -import galaxy.eggs diff --git a/lib/galaxy/eggs/__init__.py b/lib/galaxy/eggs/__init__.py index 323e17c012e..a156cc7f7f0 100644 --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -1,529 +1,7 @@ """ -Manage Galaxy eggs +For backwards compatibility """ -import ConfigParser -import glob -import HTMLParser -import os import pkg_resources -import shutil -import sys -import urllib -import urllib2 -import zipfile -import zipimport -import logging -log = logging.getLogger( __name__ ) -log.addHandler( logging.NullHandler() ) - - -galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', '..', '..' ) ) -eggs_dir = os.environ.get( 'GALAXY_EGGS_PATH', os.path.join( galaxy_dir, 'eggs' ) ) -py = 'py%s' % sys.version[:3] - - -class EggNotFetchable( Exception ): - def __init__( self, eggs ): - if type( eggs ) in ( list, tuple ): - self.eggs = eggs - else: - self.eggs = [ eggs ] - - def __str__( self ): - return ' '.join( self.eggs ) - - -# need the options to remain case sensitive -class CaseSensitiveConfigParser( ConfigParser.SafeConfigParser ): - def optionxform( self, optionstr ): - return optionstr - - -# so we can actually detect failures -class URLRetriever( urllib.FancyURLopener ): - def http_error_default( *args ): - urllib.URLopener.http_error_default( *args ) - - -class Egg( object ): - """ - Contains information about locating and downloading eggs. - """ - def __init__( self, name=None, version=None, tag=None, url=None, platform=None, crate=None ): - self.name = name - self.version = version - self.tag = tag - self.url = url - self.platform = platform - self.crate = crate - self.distribution = None - self.dir = None - self.removed_location = None - self.enable_fetch = crate.enable_egg_fetch - if self.name is not None and self.version is not None: - self.set_distribution() - - def set_dir( self ): - global eggs_dir - self.dir = eggs_dir - - def set_distribution( self ): - """ - Stores a pkg_resources Distribution object for reference later - """ - if self.dir is None: - self.set_dir() - tag = self.tag or '' - self.distribution = pkg_resources.Distribution.from_filename( - os.path.join( self.dir, '-'.join( ( self.name, self.version + tag, self.platform ) ) + '.egg' ) ) - - @property - def path( self ): - """ - Return the path of the egg, if it exists, or None - """ - if env[self.distribution.project_name]: - return env[self.distribution.project_name][0].location - return None - - def fetch( self, requirement ): - """ - fetch() serves as the install method to pkg_resources.working_set.resolve() - """ - def find_alternative(): - """ - Some platforms (e.g. Solaris) support eggs compiled on older platforms - """ - class LinkParser( HTMLParser.HTMLParser ): - """ - Finds links in what should be an Apache-style directory index - """ - def __init__( self ): - HTMLParser.HTMLParser.__init__( self ) - self.links = [] - - def handle_starttag( self, tag, attrs ): - if tag == 'a' and 'href' in dict( attrs ): - self.links.append( dict( attrs )['href'] ) - - parser = LinkParser() - try: - parser.feed( urllib2.urlopen( self.url + '/' ).read() ) - except urllib2.HTTPError, e: - if e.code == 404: - return None - parser.close() - for link in parser.links: - file = urllib.unquote( link ).rsplit( '/', 1 )[-1] - tmp_dist = pkg_resources.Distribution.from_filename( file ) - if tmp_dist.platform is not None and \ - self.distribution.project_name == tmp_dist.project_name and \ - self.distribution.version == tmp_dist.version and \ - self.distribution.py_version == tmp_dist.py_version and \ - pkg_resources.compatible_platforms( tmp_dist.platform, pkg_resources.get_platform() ): - return file - return None - - def _fetch(): - if self.url is None: - return False - if not os.path.exists( self.dir ): - os.makedirs( self.dir ) - alternative = None - try: - url = self.url + '/' + self.distribution.egg_name() + '.egg' - URLRetriever().retrieve( url, self.distribution.location ) - log.debug( "Fetched %s" % url ) - except IOError, e: - if e[1] == 404 and self.distribution.platform != py: - alternative = find_alternative() - if alternative is None: - return False - else: - return False - if alternative is not None: - try: - url = '/'.join( ( self.url, alternative ) ) - URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) ) - log.debug( "Fetched %s" % url ) - except IOError, e: - return False - self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0] - self.set_distribution() - self.unpack_if_needed() - self.remove_doppelgangers() - return True - # If being called from a version conflict, that code has removed a - # directory from sys.path. That directory could be Python's main lib - # directory (e.g. in the case of wsgiref under 2.6). Temporarily put - # it back during the execution of this code. - if self.removed_location: - sys.path.append( self.removed_location ) - try: - assert _fetch() - rval = self.distribution - except: - rval = None - if self.removed_location: - sys.path.remove( self.removed_location ) - if rval is not None: - global env - env = get_env() # reset the global Environment object now that we've obtained a new egg - return rval - - def unpack_if_needed( self ): - meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.distribution.location ) ) - if meta.has_metadata( 'not-zip-safe' ): - unpack_zipfile( self.distribution.location, self.distribution.location + "-tmp" ) - os.remove( self.distribution.location ) - os.rename( self.distribution.location + "-tmp", self.distribution.location ) - - def remove_doppelgangers( self ): - doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % ( self.name, self.platform ) ) ) - if self.distribution.location in doppelgangers: - doppelgangers.remove( self.distribution.location ) - for doppelganger in doppelgangers: - remove_file_or_path( doppelganger ) - log.debug( "Removed conflicting egg: %s" % doppelganger ) - - def resolve( self ): - try: - rval = [] - # resolve this egg and its dependencies - if self.enable_fetch: - dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) - else: - dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, lambda x: None ) - for dist in dists: - # if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them - if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ): - # TODO: like eggs.require(), this breaks requirement versioning - subdists = self.version_conflict( dist, dist.as_requirement() ) - if type( subdists ) == list: - rval.extend( subdists ) - else: - rval.append( subdists ) - else: - rval.append( dist ) - return rval - except pkg_resources.DistributionNotFound, e: - # If this statement is true, it means we do have the requested egg, - # just not one (or more) of its deps. - if e.args[0].project_name != self.distribution.project_name: - log.warning( "Warning: %s (a dependent egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) ) - return ( self.distribution, ) - else: - raise EggNotFetchable( self ) - except pkg_resources.VersionConflict, e: - if e.args[1].key != e.args[0].key: - log.error( "One of Galaxy's managed eggs depends on something which is missing, this is almost certainly a bug in the egg distribution." ) - log.error( 'Dependency "%s" requires "%s"' % ( e.args[0].project_name, str( e.args[1] ) ) ) - raise - # there's a conflicting egg on the path, remove it - return self.version_conflict( e.args[0], e.args[1] ) - - def version_conflict( self, conflict_dist, conflict_req ): - # since this conflict may be for a dependent egg, find the correct egg from the crate - if conflict_dist.project_name == self.distribution.project_name: - egg = self - dist = egg.distribution - elif conflict_dist.project_name in self.crate.all_names: - egg = self.crate[conflict_dist.project_name] - dist = egg.distribution - else: - # should not happen, but just in case - egg = None - dist = conflict_dist - # use the canonical path to locate and remove the conflict from the working set - location = os.path.realpath( conflict_dist.location ) - for entry in pkg_resources.working_set.entries: - if os.path.realpath( entry ) == location: - pkg_resources.working_set.entries.remove( entry ) - break - else: - location = entry = None - del pkg_resources.working_set.by_key[conflict_dist.key] - # remove the conflict from sys.path - if entry is not None: - pkg_resources.working_set.entry_keys[entry] = [] - if entry in sys.path: - sys.path.remove(entry) - # if the conflict is a dependent egg, fetch that specific egg - if egg: - # Store the removed path so the fetch method can use it - egg.removed_location = location - if self.enable_fetch: - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch ) - else: - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, lambda x: None ) - egg.removed_location = None - else: - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env ) - # re-add the path if it's a non-egg dir, in case more deps live there - if location is not None and not location.endswith( '.egg' ): - pkg_resources.working_set.entries.append( location ) - sys.path.append( location ) - return r - - def require( self ): - try: - dists = self.resolve() - for dist in dists: - if dist.location not in pkg_resources.working_set.entries: - pkg_resources.working_set.add( dist ) - return dists - except: - raise - - -class Crate( object ): - """ - Reads the eggs.ini file for use with checking and fetching. - """ - config_file = os.path.join( galaxy_dir, 'eggs.ini' ) - - def __init__( self, galaxy_config_file=None, platform=None ): - self.eggs = {} - self.config = CaseSensitiveConfigParser() - self.repo = None - self.no_auto = [] - self.platform = platform - self.py_platform = None - if platform is not None: - self.py_platform = platform.split( '-' )[0] - self.enable_egg_fetch = string_as_bool(os.environ.get('GALAXY_ENABLE_EGG_FETCH', True)) - self.enable_eggs = string_as_bool(os.environ.get('GALAXY_ENABLE_EGGS', True)) - self.try_dependencies_from_env = string_as_bool(os.environ.get('GALAXY_TRY_DEPENDENCIES_FROM_ENV', False)) - self.galaxy_config = GalaxyConfig( galaxy_config_file ) - self.parse() - - def parse( self ): - self.config.read( Crate.config_file ) - self.repo = self.config.get( 'general', 'repository' ) - self.no_auto = self.config.get( 'general', 'no_auto' ).split() - self.parse_egg_section( self.config.items( 'eggs:platform' ), self.config.items( 'tags' ), True ) - self.parse_egg_section( self.config.items( 'eggs:noplatform' ), self.config.items( 'tags' ) ) - - def parse_egg_section( self, eggs, tags, full_platform=False, egg_class=Egg ): - for name, version in eggs: - tag = dict( tags ).get( name, '' ) - url = '/'.join( ( self.repo, name ) ) - if full_platform: - platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) ) - else: - platform = self.py_platform or py - egg = egg_class( name=name, version=version, tag=tag, url=url, platform=platform, crate=self ) - self.eggs[name] = egg - - @property - def config_missing( self ): - """ - Return true if any eggs are missing, conditional on options set in the - Galaxy config file. - """ - for egg in self.config_eggs: - if not egg.path: - return True - return False - - @property - def all_missing( self ): - """ - Return true if any eggs in the eggs config file are missing. - """ - for egg in self.all_eggs: - if not os.path.exists( egg.distribution.location ): - return True - return False - - @property - def config_names( self ): - """ - Return a list of names of all eggs in the crate that are needed based - on the options set in the Galaxy config file. - """ - return [ egg.name for egg in self.config_eggs ] - - @property - def all_names( self ): - """ - Return a list of names of all eggs in the crate. - """ - return [ egg.name for egg in self.all_eggs ] - - @property - def config_eggs( self ): - """ - Return a list of all eggs in the crate that are needed based on the - options set in the Galaxy config file. - """ - return [ egg for egg in self.eggs.values() if self.galaxy_config.check_conditional( egg.name ) ] - - @property - def all_eggs( self ): - """ - Return a list of all eggs in the crate. - """ - rval = [] - for egg in self.eggs.values(): - if egg.name in self.galaxy_config.always_conditional and not self.galaxy_config.check_conditional( egg.name ): - continue - rval.append( egg ) - return rval - - def __getitem__( self, name ): - """ - Return a specific egg. - """ - name = name.replace( '-', '_' ) - return self.eggs[name] - - def resolve( self, all=False ): - """ - Try to resolve (e.g. fetch) all eggs in the crate. - """ - if self.enable_eggs and not self.try_dependencies_from_env: - if all: - eggs = self.all_eggs - else: - eggs = self.config_eggs - eggs = filter( lambda x: x.name not in self.no_auto, eggs ) - missing = [] - for egg in eggs: - try: - egg.resolve() - except EggNotFetchable: - missing.append( egg ) - if missing: - raise EggNotFetchable( missing ) - else: - log.info('Dependencies will attempt to be loaded from the environment') - - -class GalaxyConfig( object ): - always_conditional = ( 'pysam', 'ctypes', 'python_daemon' ) - - def __init__( self, config_file ): - if config_file is None: - self.config = None - else: - self.config = ConfigParser.ConfigParser() - if self.config.read( config_file ) == []: - raise Exception( "error: unable to read Galaxy config from %s" % config_file ) - - def check_conditional( self, egg_name ): - def check_pysam(): - # can't build pysam on solaris < 10 - plat = pkg_resources.get_platform().split( '-' ) - if plat[0] == 'solaris': - minor = plat[1].split('.')[1] - if int( minor ) < 10: - return False - return True - # If we're using require() we may not have a Galaxy config file, but if - # we're using require(), we don't care about conditionals. - if self.config is None: - return True - if egg_name == "pysqlite": - # SQLite is different since it can be specified in two config vars and defaults to True - try: - return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) - except: - return True - else: - try: - return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ), - "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), - "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), - "drmaa": lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), - "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), - "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), - "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), - "pysam": lambda: check_pysam(), - "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" - }.get( egg_name, lambda: True )() - except: - return False - - - -def string_as_bool( string ): - if str( string ).lower() in ( 'true', 'yes', 'on' ): - return True - else: - return False - -def get_env(): - env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() ) - for dist in pkg_resources.find_distributions( eggs_dir, False ): - env.add( dist ) - return env -env = get_env() - - -def require( req_str ): - c = Crate( None ) - req = pkg_resources.Requirement.parse( req_str ) - if c.try_dependencies_from_env or not c.enable_eggs: - try: - return pkg_resources.working_set.require( req_str ) - except Exception as exc: - if not c.enable_eggs: - raise - log.info("%s not found in local environment, will try Galaxy egg: %s", (req_str, exc)) - # TODO: This breaks egg version requirements. Not currently a problem, but - # it could become one. - try: - return c[req.project_name].require() - except KeyError: - # not a galaxy-owned dependency - return pkg_resources.working_set.require( req_str ) - except EggNotFetchable, e: - raise EggNotFetchable( str( [ egg.name for egg in e.eggs ] ) ) -pkg_resources.require = require - - -def unpack_zipfile( filename, extract_dir, ignores=[] ): - z = zipfile.ZipFile(filename) - try: - for info in z.infolist(): - name = info.filename - mode = (info.external_attr >> 16L) & 0777 - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - target = os.path.join(extract_dir, *name.split('/')) - if not target: - continue - for ignore in ignores: - if ignore in name: - continue - if name.endswith('/'): - # directory - pkg_resources.ensure_directory(target) - else: - # file - pkg_resources.ensure_directory(target) - data = z.read(info.filename) - f = open(target, 'wb') - try: - f.write(data) - finally: - f.close() - del data - try: - if not os.path.islink(): - os.chmod(target, mode) - except: - pass - finally: - z.close() - - -def remove_file_or_path( f ): - if os.path.isdir( f ): - shutil.rmtree( f ) - else: - os.remove( f ) +require = pkg_resources.require diff --git a/lib/galaxy/eggs/dist.py b/lib/galaxy/eggs/dist.py deleted file mode 100644 index e058d5f2a04..00000000000 --- a/lib/galaxy/eggs/dist.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -Manage Galaxy eggs -""" - -import os, sys, subprocess -from scramble import ScrambleEgg, ScrambleCrate, ScrambleFailure, galaxy_dir, py -from __init__ import Crate, CaseSensitiveConfigParser - -import logging -log = logging.getLogger( __name__ ) -log.addHandler( logging.NullHandler() ) - -class DistScrambleEgg( ScrambleEgg ): - def set_dir( self ): - self.dir = os.path.join( galaxy_dir, 'dist-eggs', self.name ) - if not os.path.exists( self.dir ): - os.makedirs( self.dir ) - @property - def path( self ): - # don't look for compatible eggs, look for exact matches - if os.path.exists( self.distribution.location ): - return self.distribution.location - return None - def run_scramble_script( self ): - log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) - # subprocessed to sterilize the env - cmd = "ssh %s 'cd %s; PYTHONPATH= %s -ES %s'" % ( self.build_host, self.buildpath, self.python, 'scramble.py' ) - log.debug( '%s(): Executing:' % sys._getframe().f_code.co_name ) - log.debug( ' %s' % cmd ) - p = subprocess.Popen( args = cmd, shell = True ) - r = p.wait() - if r != 0: - raise ScrambleFailure( "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) - def unpack_if_needed( self ): - return # do not unpack dist eggs - -class DistScrambleCrate( ScrambleCrate ): - """ - Holds eggs with info on how to build them for distribution. - """ - dist_config_file = os.path.join( galaxy_dir, 'dist-eggs.ini' ) - def __init__( self, galaxy_config_file, build_on='all' ): - self.dist_config = CaseSensitiveConfigParser() - self.build_on = build_on - ScrambleCrate.__init__( self, galaxy_config_file ) - def parse( self ): - self.dist_config.read( DistScrambleCrate.dist_config_file ) - self.hosts = dict( self.dist_config.items( 'hosts' ) ) - self.groups = dict( self.dist_config.items( 'groups' ) ) - self.ignore = dict( self.dist_config.items( 'ignore' ) ) - self.platforms = self.get_platforms( self.build_on ) - self.noplatforms = self.get_platforms( 'noplatform' ) - Crate.parse( self ) - def get_platforms( self, wanted ): - # find all the members of a group and process them - if self.groups.has_key( wanted ): - platforms = [] - for name in self.groups[wanted].split(): - for platform in self.get_platforms( name ): - if platform not in platforms: - platforms.append( platform ) - return platforms - elif self.hosts.has_key( wanted ): - return [ wanted ] - else: - raise Exception( "unknown platform: %s" % wanted ) - def parse_egg_section( self, eggs, tags, full_platform=False ): - for name, version in eggs: - self.eggs[name] = [] - tag = dict( tags ).get( name, '' ) - url = '/'.join( ( self.repo, name ) ) - try: - sources = self.config.get( 'source', name ).split() - except: - sources = [] - try: - dependencies = self.config.get( 'dependencies', name ).split() - except: - dependencies = [] - if full_platform: - platforms = self.platforms - else: - platforms = self.noplatforms - for platform in platforms: - if name in self.ignore and platform in self.ignore[name].split(): - continue - egg = DistScrambleEgg( name, version, tag, url, platform, self ) - host_info = self.hosts[platform].split() - egg.build_host, egg.python = host_info[:2] - egg.sources = sources - egg.dependencies = dependencies - self.eggs[name].append( egg ) diff --git a/lib/galaxy/eggs/scramble.py b/lib/galaxy/eggs/scramble.py deleted file mode 100644 index a22ccdc9621..00000000000 --- a/lib/galaxy/eggs/scramble.py +++ /dev/null @@ -1,248 +0,0 @@ -""" -Manage Galaxy eggs -""" - -import os, sys, shutil, tempfile, subprocess, urlparse, urllib -from __init__ import Egg, Crate, URLRetriever, galaxy_dir, py, unpack_zipfile, EggNotFetchable -from distutils.sysconfig import get_config_var - -import tarfile, zipfile, zlib -arctypes = ( 'tar.gz', 'tgz', 'zip' ) - -import logging -log = logging.getLogger( __name__ ) -log.addHandler( logging.NullHandler() ) - -import pkg_resources - -class ScrambleFailure( Exception ): - def __init__( self, eggs, msg=None ): - if type( eggs ) in ( list, tuple ): - self.eggs = eggs - else: - self.eggs = [ eggs ] - self.msg = msg - def __str__( self ): - return self.msg or ' '.join( self.eggs ) - -class ScrambleEgg( Egg ): - """ - Contains information about scrambling eggs. - """ - scramble_dir = os.path.join( galaxy_dir, 'scripts', 'scramble' ) - archive_dir = os.path.join( scramble_dir, 'archives' ) - script_dir = os.path.join( scramble_dir, 'scripts' ) - build_dir = os.path.join( scramble_dir, 'build' ) - ez_setup = os.path.join( scramble_dir, 'lib', 'ez_setup.py' ) - ez_setup_url = 'http://peak.telecommunity.com/dist/ez_setup.py' - def __init__( self, *args, **kwargs ): - Egg.__init__( self, *args, **kwargs ) - self.sources = [] - self.dependencies = [] - self.buildpath = None - self.source_path = None - self.py = py - self.build_host = None - self.python = sys.executable - def scramble( self ): - if self.path: - log.warning( "%s(): Egg already exists, remove to force rebuild:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.path ) - return - self.fetch_source() - self.unpack_source() - self.copy_build_script() - if not os.path.exists( ScrambleEgg.ez_setup ): - URLRetriever().retrieve( ScrambleEgg.ez_setup_url, ScrambleEgg.ez_setup ) - self.run_scramble_script() - new_egg = os.path.join( self.buildpath, 'dist', os.path.basename( self.distribution.location ) ) - if not os.path.exists( new_egg ): - raise ScrambleFailure( self, "%s(): Egg build for %s did not appear to fail, but no egg found to copy from expected path:\n %s" % ( sys._getframe().f_code.co_name, self.name, new_egg ) ) - shutil.copyfile( new_egg, self.distribution.location ) - log.warning( "%s(): Copied egg to:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.distribution.location ) - self.unpack_if_needed() - self.remove_doppelgangers() - # scramble helper methods - def get_tld( self, names ): - tld = names[0].split( os.path.sep, 1 )[0] - for name in names: - try: - assert tld == name.split( os.path.sep, 1 )[0] - except: - raise Exception( "%s(): Archive contains multiple top-level directories!" % sys._getframe().f_code.co_name ) - return tld - def fetch_one( self, urls ): - """ - Fetches the first available archive out of a list. - """ - for url in urls: - file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) - if os.path.exists( file ): - log.warning( "%s(): Using existing source, remove to download again:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % file ) - return file - # if we don't have one, get one - for url in urls: - file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) - try: - log.debug( "%s(): Trying to fetch:" % sys._getframe().f_code.co_name ) - log.debug( " %s" % url ) - URLRetriever().retrieve( url, file + '.download' ) - shutil.move( file + '.download', file ) - log.debug( "%s(): Fetched to:" % sys._getframe().f_code.co_name ) - log.debug( " %s" % file ) - return file - except IOError, e: - if e[1] != 404: - raise - else: - return None - def fetch_source( self ): - """ - Get egg (and dependent) source - """ - if not os.path.exists( ScrambleEgg.archive_dir ): - os.makedirs( ScrambleEgg.archive_dir ) - urls = [] - url_base = self.url + '/' + '-'.join( ( self.name, self.version ) ) - urls.extend( map( lambda x: '.'.join( ( url_base, x ) ), arctypes ) ) - if self.tag: - urls.extend( map( lambda x: '.'.join( ( url_base + self.tag, x ) ), arctypes ) ) - self.source_path = self.fetch_one( urls ) - if self.source_path is None: - raise Exception( "%s(): Couldn't find a suitable source archive for %s %s from %s" % ( sys._getframe().f_code.co_name, self.name, self.version, self.url ) ) - for url in self.sources: - if not urlparse.urlparse( url )[0]: - url = self.url + '/' + url.lstrip( '/' ) - urls = [ url ] - urls.extend( map( lambda x: '.'.join( ( url, x ) ), arctypes ) ) # allows leaving off the extension and we'll try to find one - file = self.fetch_one( urls ) - if file is None: - raise Exception( "%s(): Couldn't fetch extra source for %s, check path in %s. URL(s) attempted output above." % ( sys._getframe().f_code.co_name, self.name, Crate.config_file, ) ) - def unpack_source( self ): - unpack_dir = os.path.join( ScrambleEgg.build_dir, self.platform ) - if not os.path.exists( unpack_dir ): - os.makedirs( unpack_dir ) - self.buildpath = os.path.join( unpack_dir, self.name ) - if os.path.exists( self.buildpath ): - log.warning( "%s(): Removing old build directory at:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.buildpath ) - shutil.rmtree( self.buildpath ) - if tarfile.is_tarfile( self.source_path ): - self.unpack_tar() - elif zipfile.is_zipfile( self.source_path ): - self.unpack_zip() - else: - raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, self.source_path ) ) - log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.buildpath ) - def unpack_zip( self ): - unpack_path = os.path.dirname( self.buildpath ) - tld = self.get_tld( zipfile.ZipFile( self.source_path, 'r' ).namelist() ) - unpack_zipfile( self.source_path, unpack_path, ( 'ez_setup', ) ) - os.rename( os.path.join( unpack_path, tld ), self.buildpath ) - def unpack_tar( self ): - unpack_path = os.path.dirname( self.buildpath ) - t = tarfile.open( self.source_path, "r" ) - members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() ) - tld = self.get_tld( [ x.name for x in members ] ) - cur = os.getcwd() - os.chdir( unpack_path ) - for member in members: - try: - t.extract( member ) - except: - log.exception( "Could not extract %s", member ) - raise - t.close() - os.rename( tld, self.name ) - os.chdir( cur ) - def copy_build_script( self ): - # will try: - # bx_python-py2.4-solaris-2.11-i86pc.py - # bx_python-py2.4-solaris.py - # bx_python-solaris-2.11-i86pc.py - # bx_python-solaris.py - # bx_python-py2.4.py - # bx_python.py - # generic.py - platform = self.platform.replace( '-ucs2', '' ).replace( '-ucs4', '' ) # ucs is unimportant here - build_scripts = ( - "%s-%s.py" % ( self.name, platform ), - "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[:2] ) ), - "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[1:] ) ), - "%s-%s.py" % ( self.name, platform.split( '-' )[:2][-1] ), - "%s-%s.py" % ( self.name, platform.split( '-' )[0] ), - "%s.py" % self.name, - "generic.py" ) - for build_script in build_scripts: - build_script = os.path.join( ScrambleEgg.script_dir, build_script ) - if os.path.exists( build_script ): - log.warning( "%s(): Using build script %s" % ( sys._getframe().f_code.co_name, build_script ) ) - break - shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) ) - verfile = open( os.path.join( self.buildpath, ".galaxy_ver" ), "w" ) - verfile.write( self.version + '\n' ) - verfile.close() - if self.tag is not None: - tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" ) - tagfile.write( self.tag + '\n' ) - tagfile.close() - if self.dependencies: - depfile = open( os.path.join( self.buildpath, ".galaxy_deps" ), "w" ) - for dependency in self.dependencies: - depfile.write( dependency + '\n' ) - depfile.close() - def run_scramble_script( self ): - log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) - # subprocessed to sterilize the env - cmd = "%s %s" % ( self.python, "scramble.py" ) - log.debug( '%s(): Executing in %s:' % ( sys._getframe().f_code.co_name, self.buildpath ) ) - log.debug( ' %s' % cmd ) - p = subprocess.Popen( args = cmd, shell = True, cwd = self.buildpath ) - r = p.wait() - if r != 0: - if sys.platform == 'sunos5' and get_config_var('CC').endswith('pycc') and not os.environ.get( 'PYCC_CC', None ): - log.error( "%s(): Your python interpreter was compiled with Sun's pycc" % sys._getframe().f_code.co_name ) - log.error( " pseudo-compiler. You may need to set PYCC_CC and PYCC_CXX in your" ) - log.error( " environment if your compiler is in a non-standard location." ) - raise ScrambleFailure( self, "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) - -class ScrambleCrate( Crate ): - """ - Reads the eggs.ini file for use with scrambling eggs. - """ - def parse( self ): - Crate.parse( self ) - # get dependency sources - for egg in self.eggs.values(): - try: - egg.sources = self.config.get( "source", egg.name ).split() - except: - egg.sources = [] - try: - egg.dependencies = self.config.get( "dependencies", egg.name ).split() - except: - egg.dependencies = [] - def parse_egg_section( self, *args, **kwargs ): - kwargs['egg_class'] = ScrambleEgg - Crate.parse_egg_section( self, *args, **kwargs ) - def scramble( self, all=False ): - if all: - eggs = self.all_eggs - else: - eggs = self.config_eggs - eggs = filter( lambda x: x.name not in self.no_auto, eggs ) - failed = [] - for egg in eggs: - try: - egg.scramble() - except Exception, e: - failed.append( egg ) - last_exc = e - if failed: - if len( failed ) == 1: - raise last_exc # only 1 failure out of the crate, be more informative - else: - raise ScrambleFailure( failed ) diff --git a/lib/galaxy/wheels.py b/lib/galaxy/wheels.py new file mode 100644 index 00000000000..b1d3b2bc0be --- /dev/null +++ b/lib/galaxy/wheels.py @@ -0,0 +1,67 @@ +""" +""" + +from os import pardir +from os.path import abspath, join, dirname + +try: + import configparser +except: + import ConfigParser as configparser + + +BASE_REQUIREMENTS = [] +OPTIONAL_REQUIREMENTS = ( + 'pysqlite', + 'psycopg2', +) + +for line in open(join(abspath(dirname(__file__)), pardir, pardir, 'requirements.txt')): + line = line.strip() + try: + line = line.split()[0] + except: + continue + if line.startswith( '#' ) or line == '': + continue + BASE_REQUIREMENTS.append( line ) + + +class GalaxyConfig( object ): + def __init__( self, config_file ): + self.config = configparser.ConfigParser() + if self.config.read( config_file ) == []: + raise Exception( "error: unable to read Galaxy config from %s" % config_file ) + + def check( self, name ): + # SQLite is different since it can be specified in two config vars and defaults to True + if name == "pysqlite": + try: + return config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) + except: + return True + else: + try: + return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ), + "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), + "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), + "drmaa": lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), + "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), + "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), + "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), + "pysam": lambda: check_pysam(), + "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" + }.get( name, lambda: True )() + except: + return False + + +def requirements( config_file ): + config = GalaxyConfig( config_file ) + rval = list(BASE_REQUIREMENTS) + + for opt in OPTIONAL_REQUIREMENTS: + if config.check(opt): + rval.append( opt ) + + return rval diff --git a/lib/pkg_resources.py b/lib/pkg_resources.py deleted file mode 100644 index 79db00b8c28..00000000000 --- a/lib/pkg_resources.py +++ /dev/null @@ -1,2625 +0,0 @@ -"""Package resource API --------------------- - -A resource is a logical file contained within a package, or a logical -subdirectory thereof. The package resource API expects resource names -to have their path parts separated with ``/``, *not* whatever the local -path separator is. Do not use os.path operations to manipulate resource -names being passed into the API. - -The package resource API is designed to work with normal filesystem packages, -.egg files, and unpacked .egg files. It can also work in a limited way with -.zip files and with custom PEP 302 loaders that support the ``get_data()`` -method. -""" - -import sys, os, zipimport, time, re, imp - -try: - frozenset -except NameError: - from sets import ImmutableSet as frozenset - -# capture these to bypass sandboxing -from os import utime, rename, unlink, mkdir -from os import open as os_open -from os.path import isdir, split - - -def _bypass_ensure_directory(name, mode=0777): - # Sandbox-bypassing version of ensure_directory() - dirname, filename = split(name) - if dirname and filename and not isdir(dirname): - _bypass_ensure_directory(dirname) - mkdir(dirname, mode) - - - - - - - -_state_vars = {} - -def _declare_state(vartype, **kw): - g = globals() - for name, val in kw.iteritems(): - g[name] = val - _state_vars[name] = vartype - -def __getstate__(): - state = {} - g = globals() - for k, v in _state_vars.iteritems(): - state[k] = g['_sget_'+v](g[k]) - return state - -def __setstate__(state): - g = globals() - for k, v in state.iteritems(): - g['_sset_'+_state_vars[k]](k, g[k], v) - return state - -def _sget_dict(val): - return val.copy() - -def _sset_dict(key, ob, state): - ob.clear() - ob.update(state) - -def _sget_object(val): - return val.__getstate__() - -def _sset_object(key, ob, state): - ob.__setstate__(state) - -_sget_none = _sset_none = lambda *args: None - - - - - - -def get_supported_platform(): - """Return this platform's maximum compatible version. - - distutils.util.get_platform() normally reports the minimum version - of Mac OS X that would be required to *use* extensions produced by - distutils. But what we want when checking compatibility is to know the - version of Mac OS X that we are *running*. To allow usage of packages that - explicitly require a newer version of Mac OS X, we must also know the - current version of the OS. - - If this condition occurs for any other platform with a version in its - platform strings, this function should be extended accordingly. - """ - plat = get_build_platform(); m = macosVersionString.match(plat) - if m is not None and sys.platform == "darwin": - try: - plat = 'macosx-%s-%s' % ('.'.join(_macosx_vers()[:2]), m.group(3)) - except ValueError: - pass # not Mac OS X - return plat - - - - - - - - - - - - - - - - - - - - - -__all__ = [ - # Basic resource access and distribution/entry point discovery - 'require', 'run_script', 'get_provider', 'get_distribution', - 'load_entry_point', 'get_entry_map', 'get_entry_info', 'iter_entry_points', - 'resource_string', 'resource_stream', 'resource_filename', - 'resource_listdir', 'resource_exists', 'resource_isdir', - - # Environmental control - 'declare_namespace', 'working_set', 'add_activation_listener', - 'find_distributions', 'set_extraction_path', 'cleanup_resources', - 'get_default_cache', - - # Primary implementation classes - 'Environment', 'WorkingSet', 'ResourceManager', - 'Distribution', 'Requirement', 'EntryPoint', - - # Exceptions - 'ResolutionError','VersionConflict','DistributionNotFound','UnknownExtra', - 'ExtractionError', - - # Parsing functions and string utilities - 'parse_requirements', 'parse_version', 'safe_name', 'safe_version', - 'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections', - 'safe_extra', 'to_filename', - - # filesystem utilities - 'ensure_directory', 'normalize_path', - - # Distribution "precedence" constants - 'EGG_DIST', 'BINARY_DIST', 'SOURCE_DIST', 'CHECKOUT_DIST', 'DEVELOP_DIST', - - # "Provider" interfaces, implementations, and registration/lookup APIs - 'IMetadataProvider', 'IResourceProvider', 'FileMetadata', - 'PathMetadata', 'EggMetadata', 'EmptyProvider', 'empty_provider', - 'NullProvider', 'EggProvider', 'DefaultProvider', 'ZipProvider', - 'register_finder', 'register_namespace_handler', 'register_loader_type', - 'fixup_namespace_packages', 'get_importer', - - # Deprecated/backward compatibility only - 'run_main', 'AvailableDistributions', -] -class ResolutionError(Exception): - """Abstract base for dependency resolution errors""" - def __repr__(self): return self.__class__.__name__+repr(self.args) - -class VersionConflict(ResolutionError): - """An already-installed version conflicts with the requested version""" - -class DistributionNotFound(ResolutionError): - """A requested distribution was not found""" - -class UnknownExtra(ResolutionError): - """Distribution doesn't have an "extra feature" of the given name""" -_provider_factories = {} -PY_MAJOR = sys.version[:3] -EGG_DIST = 3 -BINARY_DIST = 2 -SOURCE_DIST = 1 -CHECKOUT_DIST = 0 -DEVELOP_DIST = -1 - -def register_loader_type(loader_type, provider_factory): - """Register `provider_factory` to make providers for `loader_type` - - `loader_type` is the type or class of a PEP 302 ``module.__loader__``, - and `provider_factory` is a function that, passed a *module* object, - returns an ``IResourceProvider`` for that module. - """ - _provider_factories[loader_type] = provider_factory - -def get_provider(moduleOrReq): - """Return an IResourceProvider for the named module or requirement""" - if isinstance(moduleOrReq,Requirement): - return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0] - try: - module = sys.modules[moduleOrReq] - except KeyError: - __import__(moduleOrReq) - module = sys.modules[moduleOrReq] - loader = getattr(module, '__loader__', None) - return _find_adapter(_provider_factories, loader)(module) - -def _macosx_vers(_cache=[]): - if not _cache: - from platform import mac_ver - _cache.append(mac_ver()[0].split('.')) - return _cache[0] - -def _macosx_arch(machine): - return {'PowerPC':'ppc', 'Power_Macintosh':'ppc'}.get(machine,machine) - -def get_build_platform(): - """Return this platform's string for platform-specific distributions - - XXX Currently this is the same as ``distutils.util.get_platform()``, but it - needs some hacks for Linux and Mac OS X. - """ - from distutils.util import get_platform - plat = get_platform() - if sys.platform == "darwin" and not plat.startswith('macosx-'): - try: - version = _macosx_vers() - machine = os.uname()[4].replace(" ", "_") - return "macosx-%d.%d-%s" % (int(version[0]), int(version[1]), - _macosx_arch(machine)) - except ValueError: - # if someone is running a non-Mac darwin system, this will fall - # through to the default implementation - pass - return plat - -macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)") -darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") -get_platform = get_build_platform # XXX backward compat - - - - - - - - - -def compatible_platforms(provided,required): - """Can code for the `provided` platform run on the `required` platform? - - Returns true if either platform is ``None``, or the platforms are equal. - - XXX Needs compatibility checks for Linux and other unixy OSes. - """ - if provided is None or required is None or provided==required: - return True # easy case - - # Mac OS X special cases - reqMac = macosVersionString.match(required) - if reqMac: - provMac = macosVersionString.match(provided) - - # is this a Mac package? - if not provMac: - # this is backwards compatibility for packages built before - # setuptools 0.6. All packages built after this point will - # use the new macosx designation. - provDarwin = darwinVersionString.match(provided) - if provDarwin: - dversion = int(provDarwin.group(1)) - macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2)) - if dversion == 7 and macosversion >= "10.3" or \ - dversion == 8 and macosversion >= "10.4": - - #import warnings - #warnings.warn("Mac eggs should be rebuilt to " - # "use the macosx designation instead of darwin.", - # category=DeprecationWarning) - return True - return False # egg isn't macosx or legacy darwin - - # are they the same major version and machine type? - if provMac.group(1) != reqMac.group(1) or \ - provMac.group(3) != reqMac.group(3): - return False - - - - # is the required OS major update >= the provided one? - if int(provMac.group(2)) > int(reqMac.group(2)): - return False - - return True - - # XXX Linux and other platforms' special cases should go here - return False - - -def run_script(dist_spec, script_name): - """Locate distribution `dist_spec` and run its `script_name` script""" - ns = sys._getframe(1).f_globals - name = ns['__name__'] - ns.clear() - ns['__name__'] = name - require(dist_spec)[0].run_script(script_name, ns) - -run_main = run_script # backward compatibility - -def get_distribution(dist): - """Return a current distribution object for a Requirement or string""" - if isinstance(dist,basestring): dist = Requirement.parse(dist) - if isinstance(dist,Requirement): dist = get_provider(dist) - if not isinstance(dist,Distribution): - raise TypeError("Expected string, Requirement, or Distribution", dist) - return dist - -def load_entry_point(dist, group, name): - """Return `name` entry point of `group` for `dist` or raise ImportError""" - return get_distribution(dist).load_entry_point(group, name) - -def get_entry_map(dist, group=None): - """Return the entry point map for `group`, or the full entry map""" - return get_distribution(dist).get_entry_map(group) - -def get_entry_info(dist, group, name): - """Return the EntryPoint object for `group`+`name`, or ``None``""" - return get_distribution(dist).get_entry_info(group, name) - - -class IMetadataProvider: - - def has_metadata(name): - """Does the package's distribution contain the named metadata?""" - - def get_metadata(name): - """The named metadata resource as a string""" - - def get_metadata_lines(name): - """Yield named metadata resource as list of non-blank non-comment lines - - Leading and trailing whitespace is stripped from each line, and lines - with ``#`` as the first non-blank character are omitted.""" - - def metadata_isdir(name): - """Is the named metadata a directory? (like ``os.path.isdir()``)""" - - def metadata_listdir(name): - """List of metadata names in the directory (like ``os.listdir()``)""" - - def run_script(script_name, namespace): - """Execute the named script in the supplied namespace dictionary""" - - - - - - - - - - - - - - - - - - - -class IResourceProvider(IMetadataProvider): - """An object that provides access to package resources""" - - def get_resource_filename(manager, resource_name): - """Return a true filesystem path for `resource_name` - - `manager` must be an ``IResourceManager``""" - - def get_resource_stream(manager, resource_name): - """Return a readable file-like object for `resource_name` - - `manager` must be an ``IResourceManager``""" - - def get_resource_string(manager, resource_name): - """Return a string containing the contents of `resource_name` - - `manager` must be an ``IResourceManager``""" - - def has_resource(resource_name): - """Does the package contain the named resource?""" - - def resource_isdir(resource_name): - """Is the named resource a directory? (like ``os.path.isdir()``)""" - - def resource_listdir(resource_name): - """List of resource names in the directory (like ``os.listdir()``)""" - - - - - - - - - - - - - - - -class WorkingSet(object): - """A collection of active distributions on sys.path (or a similar list)""" - - def __init__(self, entries=None): - """Create working set from list of path entries (default=sys.path)""" - self.entries = [] - self.entry_keys = {} - self.by_key = {} - self.callbacks = [] - - if entries is None: - entries = sys.path - - for entry in entries: - self.add_entry(entry) - - - def add_entry(self, entry): - """Add a path item to ``.entries``, finding any distributions on it - - ``find_distributions(entry, True)`` is used to find distributions - corresponding to the path entry, and they are added. `entry` is - always appended to ``.entries``, even if it is already present. - (This is because ``sys.path`` can contain the same value more than - once, and the ``.entries`` of the ``sys.path`` WorkingSet should always - equal ``sys.path``.) - """ - self.entry_keys.setdefault(entry, []) - self.entries.append(entry) - for dist in find_distributions(entry, True): - self.add(dist, entry, False) - - - def __contains__(self,dist): - """True if `dist` is the active distribution for its project""" - return self.by_key.get(dist.key) == dist - - - - - - def find(self, req): - """Find a distribution matching requirement `req` - - If there is an active distribution for the requested project, this - returns it as long as it meets the version requirement specified by - `req`. But, if there is an active distribution for the project and it - does *not* meet the `req` requirement, ``VersionConflict`` is raised. - If there is no active distribution for the requested project, ``None`` - is returned. - """ - dist = self.by_key.get(req.key) - if dist is not None and dist not in req: - raise VersionConflict(dist,req) # XXX add more info - else: - return dist - - def iter_entry_points(self, group, name=None): - """Yield entry point objects from `group` matching `name` - - If `name` is None, yields all entry points in `group` from all - distributions in the working set, otherwise only ones matching - both `group` and `name` are yielded (in distribution order). - """ - for dist in self: - entries = dist.get_entry_map(group) - if name is None: - for ep in entries.values(): - yield ep - elif name in entries: - yield entries[name] - - def run_script(self, requires, script_name): - """Locate distribution for `requires` and run `script_name` script""" - ns = sys._getframe(1).f_globals - name = ns['__name__'] - ns.clear() - ns['__name__'] = name - self.require(requires)[0].run_script(script_name, ns) - - - - def __iter__(self): - """Yield distributions for non-duplicate projects in the working set - - The yield order is the order in which the items' path entries were - added to the working set. - """ - seen = {} - for item in self.entries: - for key in self.entry_keys[item]: - if key not in seen: - seen[key]=1 - yield self.by_key[key] - - def add(self, dist, entry=None, insert=True): - """Add `dist` to working set, associated with `entry` - - If `entry` is unspecified, it defaults to the ``.location`` of `dist`. - On exit from this routine, `entry` is added to the end of the working - set's ``.entries`` (if it wasn't already present). - - `dist` is only added to the working set if it's for a project that - doesn't already have a distribution in the set. If it's added, any - callbacks registered with the ``subscribe()`` method will be called. - """ - if insert: - dist.insert_on(self.entries, entry) - - if entry is None: - entry = dist.location - keys = self.entry_keys.setdefault(entry,[]) - keys2 = self.entry_keys.setdefault(dist.location,[]) - if dist.key in self.by_key: - return # ignore hidden distros - - self.by_key[dist.key] = dist - if dist.key not in keys: - keys.append(dist.key) - if dist.key not in keys2: - keys2.append(dist.key) - self._added_new(dist) - - def resolve(self, requirements, env=None, installer=None): - """List all distributions needed to (recursively) meet `requirements` - - `requirements` must be a sequence of ``Requirement`` objects. `env`, - if supplied, should be an ``Environment`` instance. If - not supplied, it defaults to all distributions available within any - entry or distribution in the working set. `installer`, if supplied, - will be invoked with each requirement that cannot be met by an - already-installed distribution; it should return a ``Distribution`` or - ``None``. - """ - - requirements = list(requirements)[::-1] # set up the stack - processed = {} # set of processed requirements - best = {} # key -> dist - to_activate = [] - - while requirements: - req = requirements.pop(0) # process dependencies breadth-first - if req in processed: - # Ignore cyclic or redundant dependencies - continue - dist = best.get(req.key) - if dist is None: - # Find the best distribution and add it to the map - dist = self.by_key.get(req.key) - if dist is None: - if env is None: - env = Environment(self.entries) - dist = best[req.key] = env.best_match(req, self, installer) - if dist is None: - raise DistributionNotFound(req) # XXX put more info here - to_activate.append(dist) - if dist not in req: - # Oops, the "best" so far conflicts with a dependency - raise VersionConflict(dist,req) # XXX put more info here - requirements.extend(dist.requires(req.extras)[::-1]) - processed[req] = True - - return to_activate # return list of distros to activate - - def find_plugins(self, - plugin_env, full_env=None, installer=None, fallback=True - ): - """Find all activatable distributions in `plugin_env` - - Example usage:: - - distributions, errors = working_set.find_plugins( - Environment(plugin_dirlist) - ) - map(working_set.add, distributions) # add plugins+libs to sys.path - print "Couldn't load", errors # display errors - - The `plugin_env` should be an ``Environment`` instance that contains - only distributions that are in the project's "plugin directory" or - directories. The `full_env`, if supplied, should be an ``Environment`` - contains all currently-available distributions. If `full_env` is not - supplied, one is created automatically from the ``WorkingSet`` this - method is called on, which will typically mean that every directory on - ``sys.path`` will be scanned for distributions. - - `installer` is a standard installer callback as used by the - ``resolve()`` method. The `fallback` flag indicates whether we should - attempt to resolve older versions of a plugin if the newest version - cannot be resolved. - - This method returns a 2-tuple: (`distributions`, `error_info`), where - `distributions` is a list of the distributions found in `plugin_env` - that were loadable, along with any other distributions that are needed - to resolve their dependencies. `error_info` is a dictionary mapping - unloadable plugin distributions to an exception instance describing the - error that occurred. Usually this will be a ``DistributionNotFound`` or - ``VersionConflict`` instance. - """ - - plugin_projects = list(plugin_env) - plugin_projects.sort() # scan project names in alphabetic order - - error_info = {} - distributions = {} - - if full_env is None: - env = Environment(self.entries) - env += plugin_env - else: - env = full_env + plugin_env - - shadow_set = self.__class__([]) - map(shadow_set.add, self) # put all our entries in shadow_set - - for project_name in plugin_projects: - - for dist in plugin_env[project_name]: - - req = [dist.as_requirement()] - - try: - resolvees = shadow_set.resolve(req, env, installer) - - except ResolutionError,v: - error_info[dist] = v # save error info - if fallback: - continue # try the next older version of project - else: - break # give up on this project, keep going - - else: - map(shadow_set.add, resolvees) - distributions.update(dict.fromkeys(resolvees)) - - # success, no need to try any more versions of this project - break - - distributions = list(distributions) - distributions.sort() - - return distributions, error_info - - - - - - def require(self, *requirements): - """Ensure that distributions matching `requirements` are activated - - `requirements` must be a string or a (possibly-nested) sequence - thereof, specifying the distributions and versions required. The - return value is a sequence of the distributions that needed to be - activated to fulfill the requirements; all relevant distributions are - included, even if they were already activated in this working set. - """ - needed = self.resolve(parse_requirements(requirements)) - - for dist in needed: - self.add(dist) - - return needed - - def subscribe(self, callback): - """Invoke `callback` for all distributions (including existing ones)""" - if callback in self.callbacks: - return - self.callbacks.append(callback) - for dist in self: - callback(dist) - - def _added_new(self, dist): - for callback in self.callbacks: - callback(dist) - - def __getstate__(self): - return ( - self.entries[:], self.entry_keys.copy(), self.by_key.copy(), - self.callbacks[:] - ) - - def __setstate__(self, (entries, keys, by_key, callbacks)): - self.entries = entries[:] - self.entry_keys = keys.copy() - self.by_key = by_key.copy() - self.callbacks = callbacks[:] - - -class Environment(object): - """Searchable snapshot of distributions on a search path""" - - def __init__(self, search_path=None, platform=get_supported_platform(), python=PY_MAJOR): - """Snapshot distributions available on a search path - - Any distributions found on `search_path` are added to the environment. - `search_path` should be a sequence of ``sys.path`` items. If not - supplied, ``sys.path`` is used. - - `platform` is an optional string specifying the name of the platform - that platform-specific distributions must be compatible with. If - unspecified, it defaults to the current platform. `python` is an - optional string naming the desired version of Python (e.g. ``'2.4'``); - it defaults to the current version. - - You may explicitly set `platform` (and/or `python`) to ``None`` if you - wish to map *all* distributions, not just those compatible with the - running platform or Python version. - """ - self._distmap = {} - self._cache = {} - self.platform = platform - self.python = python - self.scan(search_path) - - def can_add(self, dist): - """Is distribution `dist` acceptable for this environment? - - The distribution must match the platform and python version - requirements specified when this environment was created, or False - is returned. - """ - return (self.python is None or dist.py_version is None - or dist.py_version==self.python) \ - and compatible_platforms(dist.platform,self.platform) - - def remove(self, dist): - """Remove `dist` from the environment""" - self._distmap[dist.key].remove(dist) - - def scan(self, search_path=None): - """Scan `search_path` for distributions usable in this environment - - Any distributions found are added to the environment. - `search_path` should be a sequence of ``sys.path`` items. If not - supplied, ``sys.path`` is used. Only distributions conforming to - the platform/python version defined at initialization are added. - """ - if search_path is None: - search_path = sys.path - - for item in search_path: - for dist in find_distributions(item): - self.add(dist) - - def __getitem__(self,project_name): - """Return a newest-to-oldest list of distributions for `project_name` - """ - try: - return self._cache[project_name] - except KeyError: - project_name = project_name.lower() - if project_name not in self._distmap: - return [] - - if project_name not in self._cache: - dists = self._cache[project_name] = self._distmap[project_name] - _sort_dists(dists) - - return self._cache[project_name] - - def add(self,dist): - """Add `dist` if we ``can_add()`` it and it isn't already added""" - if self.can_add(dist) and dist.has_version(): - dists = self._distmap.setdefault(dist.key,[]) - if dist not in dists: - dists.append(dist) - if dist.key in self._cache: - _sort_dists(self._cache[dist.key]) - - - def best_match(self, req, working_set, installer=None): - """Find distribution best matching `req` and usable on `working_set` - - This calls the ``find(req)`` method of the `working_set` to see if a - suitable distribution is already active. (This may raise - ``VersionConflict`` if an unsuitable version of the project is already - active in the specified `working_set`.) If a suitable distribution - isn't active, this method returns the newest distribution in the - environment that meets the ``Requirement`` in `req`. If no suitable - distribution is found, and `installer` is supplied, then the result of - calling the environment's ``obtain(req, installer)`` method will be - returned. - """ - dist = working_set.find(req) - if dist is not None: - return dist - for dist in self[req.key]: - if dist in req: - return dist - return self.obtain(req, installer) # try and download/install - - def obtain(self, requirement, installer=None): - """Obtain a distribution matching `requirement` (e.g. via download) - - Obtain a distro that matches requirement (e.g. via download). In the - base ``Environment`` class, this routine just returns - ``installer(requirement)``, unless `installer` is None, in which case - None is returned instead. This method is a hook that allows subclasses - to attempt other ways of obtaining a distribution before falling back - to the `installer` argument.""" - if installer is not None: - return installer(requirement) - - def __iter__(self): - """Yield the unique project names of the available distributions""" - for key in self._distmap.keys(): - if self[key]: yield key - - - - - def __iadd__(self, other): - """In-place addition of a distribution or environment""" - if isinstance(other,Distribution): - self.add(other) - elif isinstance(other,Environment): - for project in other: - for dist in other[project]: - self.add(dist) - else: - raise TypeError("Can't add %r to environment" % (other,)) - return self - - def __add__(self, other): - """Add an environment or distribution to an environment""" - new = self.__class__([], platform=None, python=None) - for env in self, other: - new += env - return new - - -AvailableDistributions = Environment # XXX backward compatibility - - -class ExtractionError(RuntimeError): - """An error occurred extracting a resource - - The following attributes are available from instances of this exception: - - manager - The resource manager that raised this exception - - cache_path - The base directory for resource extraction - - original_error - The exception instance that caused extraction to fail - """ - - - - -class ResourceManager: - """Manage resource extraction and packages""" - extraction_path = None - - def __init__(self): - self.cached_files = {} - - def resource_exists(self, package_or_requirement, resource_name): - """Does the named resource exist?""" - return get_provider(package_or_requirement).has_resource(resource_name) - - def resource_isdir(self, package_or_requirement, resource_name): - """Is the named resource an existing directory?""" - return get_provider(package_or_requirement).resource_isdir( - resource_name - ) - - def resource_filename(self, package_or_requirement, resource_name): - """Return a true filesystem path for specified resource""" - return get_provider(package_or_requirement).get_resource_filename( - self, resource_name - ) - - def resource_stream(self, package_or_requirement, resource_name): - """Return a readable file-like object for specified resource""" - return get_provider(package_or_requirement).get_resource_stream( - self, resource_name - ) - - def resource_string(self, package_or_requirement, resource_name): - """Return specified resource as a string""" - return get_provider(package_or_requirement).get_resource_string( - self, resource_name - ) - - def resource_listdir(self, package_or_requirement, resource_name): - """List the contents of the named resource directory""" - return get_provider(package_or_requirement).resource_listdir( - resource_name - ) - - def extraction_error(self): - """Give an error message for problems extracting file(s)""" - - old_exc = sys.exc_info()[1] - cache_path = self.extraction_path or get_default_cache() - - err = ExtractionError("""Can't extract file(s) to egg cache - -The following error occurred while trying to extract file(s) to the Python egg -cache: - - %s - -The Python egg cache directory is currently set to: - - %s - -Perhaps your account does not have write access to this directory? You can -change the cache directory by setting the PYTHON_EGG_CACHE environment -variable to point to an accessible directory. -""" % (old_exc, cache_path) - ) - err.manager = self - err.cache_path = cache_path - err.original_error = old_exc - raise err - - - - - - - - - - - - - - - - def get_cache_path(self, archive_name, names=()): - """Return absolute location in cache for `archive_name` and `names` - - The parent directory of the resulting path will be created if it does - not already exist. `archive_name` should be the base filename of the - enclosing egg (which may not be the name of the enclosing zipfile!), - including its ".egg" extension. `names`, if provided, should be a - sequence of path name parts "under" the egg's extraction location. - - This method should only be called by resource providers that need to - obtain an extraction location, and only for names they intend to - extract, as it tracks the generated names for possible cleanup later. - """ - extract_path = self.extraction_path or get_default_cache() - target_path = os.path.join(extract_path, archive_name+'-tmp', *names) - try: - _bypass_ensure_directory(target_path) - except: - self.extraction_error() - - self.cached_files[target_path] = 1 - return target_path - - - - - - - - - - - - - - - - - - - - def postprocess(self, tempname, filename): - """Perform any platform-specific postprocessing of `tempname` - - This is where Mac header rewrites should be done; other platforms don't - have anything special they should do. - - Resource providers should call this method ONLY after successfully - extracting a compressed resource. They must NOT call it on resources - that are already in the filesystem. - - `tempname` is the current (temporary) name of the file, and `filename` - is the name it will be renamed to by the caller after this routine - returns. - """ - - if os.name == 'posix': - # Make the resource executable - mode = ((os.stat(tempname).st_mode) | 0555) & 07777 - os.chmod(tempname, mode) - - - - - - - - - - - - - - - - - - - - - - - def set_extraction_path(self, path): - """Set the base path where resources will be extracted to, if needed. - - If you do not call this routine before any extractions take place, the - path defaults to the return value of ``get_default_cache()``. (Which - is based on the ``PYTHON_EGG_CACHE`` environment variable, with various - platform-specific fallbacks. See that routine's documentation for more - details.) - - Resources are extracted to subdirectories of this path based upon - information given by the ``IResourceProvider``. You may set this to a - temporary directory, but then you must call ``cleanup_resources()`` to - delete the extracted files when done. There is no guarantee that - ``cleanup_resources()`` will be able to remove all extracted files. - - (Note: you may not change the extraction path for a given resource - manager once resources have been extracted, unless you first call - ``cleanup_resources()``.) - """ - if self.cached_files: - raise ValueError( - "Can't change extraction path, files already extracted" - ) - - self.extraction_path = path - - def cleanup_resources(self, force=False): - """ - Delete all extracted resource files and directories, returning a list - of the file and directory names that could not be successfully removed. - This function does not have any concurrency protection, so it should - generally only be called when the extraction path is a temporary - directory exclusive to a single process. This method is not - automatically called; you must call it explicitly or register it as an - ``atexit`` function if you wish to ensure cleanup of a temporary - directory used for extractions. - """ - # XXX - - - -def get_default_cache(): - """Determine the default cache location - - This returns the ``PYTHON_EGG_CACHE`` environment variable, if set. - Otherwise, on Windows, it returns a "Python-Eggs" subdirectory of the - "Application Data" directory. On all other systems, it's "~/.python-eggs". - """ - try: - return os.environ['PYTHON_EGG_CACHE'] - except KeyError: - pass - - if os.name!='nt': - return os.path.expanduser('~/.python-eggs') - - app_data = 'Application Data' # XXX this may be locale-specific! - app_homes = [ - (('APPDATA',), None), # best option, should be locale-safe - (('USERPROFILE',), app_data), - (('HOMEDRIVE','HOMEPATH'), app_data), - (('HOMEPATH',), app_data), - (('HOME',), None), - (('WINDIR',), app_data), # 95/98/ME - ] - - for keys, subdir in app_homes: - dirname = '' - for key in keys: - if key in os.environ: - dirname = os.path.join(dirname, os.environ[key]) - else: - break - else: - if subdir: - dirname = os.path.join(dirname,subdir) - return os.path.join(dirname, 'Python-Eggs') - else: - raise RuntimeError( - "Please set the PYTHON_EGG_CACHE enviroment variable" - ) - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """Convert an arbitrary string to a standard version string - - Spaces become dots, and all other non-alphanumeric characters become - dashes, with runs of multiple dashes condensed to a single dash. - """ - version = version.replace(' ','.') - return re.sub('[^A-Za-z0-9.]+', '-', version) - - -def safe_extra(extra): - """Convert an arbitrary string to a standard 'extra' name - - Any runs of non-alphanumeric characters are replaced with a single '_', - and the result is always lowercased. - """ - return re.sub('[^A-Za-z0-9.]+', '_', extra).lower() - - -def to_filename(name): - """Convert a project or version name to its filename-escaped form - - Any '-' characters are currently replaced with '_'. - """ - return name.replace('-','_') - - - - - - - - -class NullProvider: - """Try to implement resources and metadata for arbitrary PEP 302 loaders""" - - egg_name = None - egg_info = None - loader = None - - def __init__(self, module): - self.loader = getattr(module, '__loader__', None) - self.module_path = os.path.dirname(getattr(module, '__file__', '')) - - def get_resource_filename(self, manager, resource_name): - return self._fn(self.module_path, resource_name) - - def get_resource_stream(self, manager, resource_name): - return StringIO(self.get_resource_string(manager, resource_name)) - - def get_resource_string(self, manager, resource_name): - return self._get(self._fn(self.module_path, resource_name)) - - def has_resource(self, resource_name): - return self._has(self._fn(self.module_path, resource_name)) - - def has_metadata(self, name): - return self.egg_info and self._has(self._fn(self.egg_info,name)) - - def get_metadata(self, name): - if not self.egg_info: - return "" - return self._get(self._fn(self.egg_info,name)) - - def get_metadata_lines(self, name): - return yield_lines(self.get_metadata(name)) - - def resource_isdir(self,resource_name): - return self._isdir(self._fn(self.module_path, resource_name)) - - def metadata_isdir(self,name): - return self.egg_info and self._isdir(self._fn(self.egg_info,name)) - - - def resource_listdir(self,resource_name): - return self._listdir(self._fn(self.module_path,resource_name)) - - def metadata_listdir(self,name): - if self.egg_info: - return self._listdir(self._fn(self.egg_info,name)) - return [] - - def run_script(self,script_name,namespace): - script = 'scripts/'+script_name - if not self.has_metadata(script): - raise ResolutionError("No script named %r" % script_name) - script_text = self.get_metadata(script).replace('\r\n','\n') - script_text = script_text.replace('\r','\n') - script_filename = self._fn(self.egg_info,script) - namespace['__file__'] = script_filename - if os.path.exists(script_filename): - execfile(script_filename, namespace, namespace) - else: - from linecache import cache - cache[script_filename] = ( - len(script_text), 0, script_text.split('\n'), script_filename - ) - script_code = compile(script_text,script_filename,'exec') - exec script_code in namespace, namespace - - def _has(self, path): - raise NotImplementedError( - "Can't perform this operation for unregistered loader type" - ) - - def _isdir(self, path): - raise NotImplementedError( - "Can't perform this operation for unregistered loader type" - ) - - def _listdir(self, path): - raise NotImplementedError( - "Can't perform this operation for unregistered loader type" - ) - - def _fn(self, base, resource_name): - if resource_name: - return os.path.join(base, *resource_name.split('/')) - return base - - def _get(self, path): - if hasattr(self.loader, 'get_data'): - return self.loader.get_data(path) - raise NotImplementedError( - "Can't perform this operation for loaders without 'get_data()'" - ) - -register_loader_type(object, NullProvider) - - -class EggProvider(NullProvider): - """Provider based on a virtual filesystem""" - - def __init__(self,module): - NullProvider.__init__(self,module) - self._setup_prefix() - - def _setup_prefix(self): - # we assume here that our metadata may be nested inside a "basket" - # of multiple eggs; that's why we use module_path instead of .archive - path = self.module_path - old = None - while path!=old: - if path.lower().endswith('.egg'): - self.egg_name = os.path.basename(path) - self.egg_info = os.path.join(path, 'EGG-INFO') - self.egg_root = path - break - old = path - path, base = os.path.split(path) - - - - - - -class DefaultProvider(EggProvider): - """Provides access to package resources in the filesystem""" - - def _has(self, path): - return os.path.exists(path) - - def _isdir(self,path): - return os.path.isdir(path) - - def _listdir(self,path): - return os.listdir(path) - - def get_resource_stream(self, manager, resource_name): - return open(self._fn(self.module_path, resource_name), 'rb') - - def _get(self, path): - stream = open(path, 'rb') - try: - return stream.read() - finally: - stream.close() - -register_loader_type(type(None), DefaultProvider) - - -class EmptyProvider(NullProvider): - """Provider that returns nothing for all requests""" - - _isdir = _has = lambda self,path: False - _get = lambda self,path: '' - _listdir = lambda self,path: [] - module_path = None - - def __init__(self): - pass - -empty_provider = EmptyProvider() - - - - -class ZipProvider(EggProvider): - """Resource support for zips and eggs""" - - eagers = None - - def __init__(self, module): - EggProvider.__init__(self,module) - self.zipinfo = zipimport._zip_directory_cache[self.loader.archive] - self.zip_pre = self.loader.archive+os.sep - - def _zipinfo_name(self, fspath): - # Convert a virtual filename (full path to file) into a zipfile subpath - # usable with the zipimport directory cache for our target archive - if fspath.startswith(self.zip_pre): - return fspath[len(self.zip_pre):] - raise AssertionError( - "%s is not a subpath of %s" % (fspath,self.zip_pre) - ) - - def _parts(self,zip_path): - # Convert a zipfile subpath into an egg-relative path part list - fspath = self.zip_pre+zip_path # pseudo-fs path - if fspath.startswith(self.egg_root+os.sep): - return fspath[len(self.egg_root)+1:].split(os.sep) - raise AssertionError( - "%s is not a subpath of %s" % (fspath,self.egg_root) - ) - - def get_resource_filename(self, manager, resource_name): - if not self.egg_name: - raise NotImplementedError( - "resource_filename() only supported for .egg, not .zip" - ) - # no need to lock for extraction, since we use temp names - zip_path = self._resource_to_zip(resource_name) - eagers = self._get_eager_resources() - if '/'.join(self._parts(zip_path)) in eagers: - for name in eagers: - self._extract_resource(manager, self._eager_to_zip(name)) - return self._extract_resource(manager, zip_path) - - def _extract_resource(self, manager, zip_path): - - if zip_path in self._index(): - for name in self._index()[zip_path]: - last = self._extract_resource( - manager, os.path.join(zip_path, name) - ) - return os.path.dirname(last) # return the extracted directory name - - zip_stat = self.zipinfo[zip_path] - t,d,size = zip_stat[5], zip_stat[6], zip_stat[3] - date_time = ( - (d>>9)+1980, (d>>5)&0xF, d&0x1F, # ymd - (t&0xFFFF)>>11, (t>>5)&0x3F, (t&0x1F) * 2, 0, 0, -1 # hms, etc. - ) - timestamp = time.mktime(date_time) - - try: - real_path = manager.get_cache_path( - self.egg_name, self._parts(zip_path) - ) - - if os.path.isfile(real_path): - stat = os.stat(real_path) - if stat.st_size==size and stat.st_mtime==timestamp: - # size and stamp match, don't bother extracting - return real_path - - outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path)) - os.write(outf, self.loader.get_data(zip_path)) - os.close(outf) - utime(tmpnam, (timestamp,timestamp)) - manager.postprocess(tmpnam, real_path) - - try: - rename(tmpnam, real_path) - - except os.error: - if os.path.isfile(real_path): - stat = os.stat(real_path) - - if stat.st_size==size and stat.st_mtime==timestamp: - # size and stamp match, somebody did it just ahead of - # us, so we're done - return real_path - elif os.name=='nt': # Windows, del old file and retry - unlink(real_path) - rename(tmpnam, real_path) - return real_path - raise - - except os.error: - manager.extraction_error() # report a user-friendly error - - return real_path - - def _get_eager_resources(self): - if self.eagers is None: - eagers = [] - for name in ('native_libs.txt', 'eager_resources.txt'): - if self.has_metadata(name): - eagers.extend(self.get_metadata_lines(name)) - self.eagers = eagers - return self.eagers - - def _index(self): - try: - return self._dirindex - except AttributeError: - ind = {} - for path in self.zipinfo: - parts = path.split(os.sep) - while parts: - parent = os.sep.join(parts[:-1]) - if parent in ind: - ind[parent].append(parts[-1]) - break - else: - ind[parent] = [parts.pop()] - self._dirindex = ind - return ind - - def _has(self, fspath): - zip_path = self._zipinfo_name(fspath) - return zip_path in self.zipinfo or zip_path in self._index() - - def _isdir(self,fspath): - return self._zipinfo_name(fspath) in self._index() - - def _listdir(self,fspath): - return list(self._index().get(self._zipinfo_name(fspath), ())) - - def _eager_to_zip(self,resource_name): - return self._zipinfo_name(self._fn(self.egg_root,resource_name)) - - def _resource_to_zip(self,resource_name): - return self._zipinfo_name(self._fn(self.module_path,resource_name)) - -register_loader_type(zipimport.zipimporter, ZipProvider) - - - - - - - - - - - - - - - - - - - - - - - - -class FileMetadata(EmptyProvider): - """Metadata handler for standalone PKG-INFO files - - Usage:: - - metadata = FileMetadata("/path/to/PKG-INFO") - - This provider rejects all data and metadata requests except for PKG-INFO, - which is treated as existing, and will be the contents of the file at - the provided location. - """ - - def __init__(self,path): - self.path = path - - def has_metadata(self,name): - return name=='PKG-INFO' - - def get_metadata(self,name): - if name=='PKG-INFO': - return open(self.path,'rU').read() - raise KeyError("No metadata except PKG-INFO is available") - - def get_metadata_lines(self,name): - return yield_lines(self.get_metadata(name)) - - - - - - - - - - - - - - - - -class PathMetadata(DefaultProvider): - """Metadata provider for egg directories - - Usage:: - - # Development eggs: - - egg_info = "/path/to/PackageName.egg-info" - base_dir = os.path.dirname(egg_info) - metadata = PathMetadata(base_dir, egg_info) - dist_name = os.path.splitext(os.path.basename(egg_info))[0] - dist = Distribution(basedir,project_name=dist_name,metadata=metadata) - - # Unpacked egg directories: - - egg_path = "/path/to/PackageName-ver-pyver-etc.egg" - metadata = PathMetadata(egg_path, os.path.join(egg_path,'EGG-INFO')) - dist = Distribution.from_filename(egg_path, metadata=metadata) - """ - - def __init__(self, path, egg_info): - self.module_path = path - self.egg_info = egg_info - - -class EggMetadata(ZipProvider): - """Metadata provider for .egg files""" - - def __init__(self, importer): - """Create a metadata provider from a zipimporter""" - - self.zipinfo = zipimport._zip_directory_cache[importer.archive] - self.zip_pre = importer.archive+os.sep - self.loader = importer - if importer.prefix: - self.module_path = os.path.join(importer.archive, importer.prefix) - else: - self.module_path = importer.archive - self._setup_prefix() - - -class ImpWrapper: - """PEP 302 Importer that wraps Python's "normal" import algorithm""" - - def __init__(self, path=None): - self.path = path - - def find_module(self, fullname, path=None): - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [self.path] - try: - file, filename, etc = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(file, filename, etc) - - -class ImpLoader: - """PEP 302 Loader that wraps Python's "normal" import algorithm""" - - def __init__(self, file, filename, etc): - self.file = file - self.filename = filename - self.etc = etc - - def load_module(self, fullname): - try: - mod = imp.load_module(fullname, self.file, self.filename, self.etc) - finally: - if self.file: self.file.close() - # Note: we don't set __loader__ because we want the module to look - # normal; i.e. this is just a wrapper for standard import machinery - return mod - - - - -def get_importer(path_item): - """Retrieve a PEP 302 "importer" for the given path item - - If there is no importer, this returns a wrapper around the builtin import - machinery. The returned importer is only cached if it was created by a - path hook. - """ - try: - importer = sys.path_importer_cache[path_item] - except KeyError: - for hook in sys.path_hooks: - try: - importer = hook(path_item) - except ImportError: - pass - else: - break - else: - importer = None - - sys.path_importer_cache.setdefault(path_item,importer) - if importer is None: - try: - importer = ImpWrapper(path_item) - except ImportError: - pass - return importer - -try: - from pkgutil import get_importer, ImpImporter -except ImportError: - pass # Python 2.3 or 2.4, use our own implementation -else: - ImpWrapper = ImpImporter # Python 2.5, use pkgutil's implementation - del ImpLoader, ImpImporter - - - - - - -_declare_state('dict', _distribution_finders = {}) - -def register_finder(importer_type, distribution_finder): - """Register `distribution_finder` to find distributions in sys.path items - - `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item - handler), and `distribution_finder` is a callable that, passed a path - item and the importer instance, yields ``Distribution`` instances found on - that path item. See ``pkg_resources.find_on_path`` for an example.""" - _distribution_finders[importer_type] = distribution_finder - - -def find_distributions(path_item, only=False): - """Yield distributions accessible via `path_item`""" - importer = get_importer(path_item) - finder = _find_adapter(_distribution_finders, importer) - return finder(importer, path_item, only) - -def find_in_zip(importer, path_item, only=False): - metadata = EggMetadata(importer) - if metadata.has_metadata('PKG-INFO'): - yield Distribution.from_filename(path_item, metadata=metadata) - if only: - return # don't yield nested distros - for subitem in metadata.resource_listdir('/'): - if subitem.endswith('.egg'): - subpath = os.path.join(path_item, subitem) - for dist in find_in_zip(zipimport.zipimporter(subpath), subpath): - yield dist - -register_finder(zipimport.zipimporter, find_in_zip) - -def StringIO(*args, **kw): - """Thunk to load the real StringIO on demand""" - global StringIO - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - return StringIO(*args,**kw) - -def find_nothing(importer, path_item, only=False): - return () -register_finder(object,find_nothing) - -def find_on_path(importer, path_item, only=False): - """Yield distributions accessible on a sys.path directory""" - path_item = _normalize_cached(path_item) - - if os.path.isdir(path_item) and os.access(path_item, os.R_OK): - if path_item.lower().endswith('.egg'): - # unpacked egg - yield Distribution.from_filename( - path_item, metadata=PathMetadata( - path_item, os.path.join(path_item,'EGG-INFO') - ) - ) - else: - # scan for .egg and .egg-info in directory - for entry in os.listdir(path_item): - lower = entry.lower() - if lower.endswith('.egg-info'): - fullpath = os.path.join(path_item, entry) - if os.path.isdir(fullpath): - # egg-info directory, allow getting metadata - metadata = PathMetadata(path_item, fullpath) - else: - metadata = FileMetadata(fullpath) - yield Distribution.from_location( - path_item,entry,metadata,precedence=DEVELOP_DIST - ) - elif not only and lower.endswith('.egg'): - for dist in find_distributions(os.path.join(path_item, entry)): - yield dist - elif not only and lower.endswith('.egg-link'): - for line in file(os.path.join(path_item, entry)): - if not line.strip(): continue - for item in find_distributions(os.path.join(path_item,line.rstrip())): - yield item - break -register_finder(ImpWrapper,find_on_path) - -_declare_state('dict', _namespace_handlers = {}) -_declare_state('dict', _namespace_packages = {}) - -def register_namespace_handler(importer_type, namespace_handler): - """Register `namespace_handler` to declare namespace packages - - `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item - handler), and `namespace_handler` is a callable like this:: - - def namespace_handler(importer,path_entry,moduleName,module): - # return a path_entry to use for child packages - - Namespace handlers are only called if the importer object has already - agreed that it can handle the relevant path item, and they should only - return a subpath if the module __path__ does not already contain an - equivalent subpath. For an example namespace handler, see - ``pkg_resources.file_ns_handler``. - """ - _namespace_handlers[importer_type] = namespace_handler - -def _handle_ns(packageName, path_item): - """Ensure that named package includes a subpath of path_item (if needed)""" - importer = get_importer(path_item) - if importer is None: - return None - loader = importer.find_module(packageName) - if loader is None: - return None - module = sys.modules.get(packageName) - if module is None: - module = sys.modules[packageName] = imp.new_module(packageName) - module.__path__ = []; _set_parent_ns(packageName) - elif not hasattr(module,'__path__'): - raise TypeError("Not a package:", packageName) - handler = _find_adapter(_namespace_handlers, importer) - subpath = handler(importer,path_item,packageName,module) - if subpath is not None: - path = module.__path__; path.append(subpath) - loader.load_module(packageName); module.__path__ = path - return subpath - -def declare_namespace(packageName): - """Declare that package 'packageName' is a namespace package""" - - imp.acquire_lock() - try: - if packageName in _namespace_packages: - return - - path, parent = sys.path, None - if '.' in packageName: - parent = '.'.join(packageName.split('.')[:-1]) - declare_namespace(parent) - __import__(parent) - try: - path = sys.modules[parent].__path__ - except AttributeError: - raise TypeError("Not a package:", parent) - - # Track what packages are namespaces, so when new path items are added, - # they can be updated - _namespace_packages.setdefault(parent,[]).append(packageName) - _namespace_packages.setdefault(packageName,[]) - - for path_item in path: - # Ensure all the parent's path items are reflected in the child, - # if they apply - _handle_ns(packageName, path_item) - - finally: - imp.release_lock() - -def fixup_namespace_packages(path_item, parent=None): - """Ensure that previously-declared namespace packages include path_item""" - imp.acquire_lock() - try: - for package in _namespace_packages.get(parent,()): - subpath = _handle_ns(package, path_item) - if subpath: fixup_namespace_packages(subpath,package) - finally: - imp.release_lock() - -def file_ns_handler(importer, path_item, packageName, module): - """Compute an ns-package subpath for a filesystem or zipfile importer""" - - subpath = os.path.join(path_item, packageName.split('.')[-1]) - normalized = _normalize_cached(subpath) - for item in module.__path__: - if _normalize_cached(item)==normalized: - break - else: - # Only return the path if it's not already there - return subpath - -register_namespace_handler(ImpWrapper,file_ns_handler) -register_namespace_handler(zipimport.zipimporter,file_ns_handler) - - -def null_ns_handler(importer, path_item, packageName, module): - return None - -register_namespace_handler(object,null_ns_handler) - - -def normalize_path(filename): - """Normalize a file/dir name for comparison purposes""" - return os.path.normcase(os.path.realpath(filename)) - -def _normalize_cached(filename,_cache={}): - try: - return _cache[filename] - except KeyError: - _cache[filename] = result = normalize_path(filename) - return result - -def _set_parent_ns(packageName): - parts = packageName.split('.') - name = parts.pop() - if parts: - parent = '.'.join(parts) - setattr(sys.modules[parent], name, sys.modules[packageName]) - - -def yield_lines(strs): - """Yield non-empty/non-comment lines of a ``basestring`` or sequence""" - if isinstance(strs,basestring): - for s in strs.splitlines(): - s = s.strip() - if s and not s.startswith('#'): # skip blank lines/comments - yield s - else: - for ss in strs: - for s in yield_lines(ss): - yield s - -LINE_END = re.compile(r"\s*(#.*)?$").match # whitespace and comment -CONTINUE = re.compile(r"\s*\\\s*(#.*)?$").match # line continuation -DISTRO = re.compile(r"\s*((\w|[-.])+)").match # Distribution or extra -VERSION = re.compile(r"\s*(<=?|>=?|==|!=)\s*((\w|[-.])+)").match # ver. info -COMMA = re.compile(r"\s*,").match # comma between items -OBRACKET = re.compile(r"\s*\[").match -CBRACKET = re.compile(r"\s*\]").match -MODULE = re.compile(r"\w+(\.\w+)*$").match -EGG_NAME = re.compile( - r"(?P[^-]+)" - r"( -(?P[^-]+) (-py(?P[^-]+) (-(?P.+))? )? )?", - re.VERBOSE | re.IGNORECASE -).match - -component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) -replace = {'pre':'c', 'preview':'c','-':'final-','rc':'c','dev':'@'}.get - -def _parse_version_parts(s): - for part in component_re.split(s): - part = replace(part,part) - if not part or part=='.': - continue - if part[:1] in '0123456789': - yield part.zfill(8) # pad for numeric comparison - else: - yield '*'+part - - yield '*final' # ensure that alpha/beta/candidate are before final - -def parse_version(s): - """Convert a version string to a chronologically-sortable key - - This is a rough cross between distutils' StrictVersion and LooseVersion; - if you give it versions that would work with StrictVersion, then it behaves - the same; otherwise it acts like a slightly-smarter LooseVersion. It is - *possible* to create pathological version coding schemes that will fool - this parser, but they should be very rare in practice. - - The returned value will be a tuple of strings. Numeric portions of the - version are padded to 8 digits so they will compare numerically, but - without relying on how numbers compare relative to strings. Dots are - dropped, but dashes are retained. Trailing zeros between alpha segments - or dashes are suppressed, so that e.g. "2.4.0" is considered the same as - "2.4". Alphanumeric parts are lower-cased. - - The algorithm assumes that strings like "-" and any alpha string that - alphabetically follows "final" represents a "patch level". So, "2.4-1" - is assumed to be a branch or patch of "2.4", and therefore "2.4.1" is - considered newer than "2.4-1", which in turn is newer than "2.4". - - Strings like "a", "b", "c", "alpha", "beta", "candidate" and so on (that - come before "final" alphabetically) are assumed to be pre-release versions, - so that the version "2.4" is considered newer than "2.4a1". - - Finally, to handle miscellaneous cases, the strings "pre", "preview", and - "rc" are treated as if they were "c", i.e. as though they were release - candidates, and therefore are not as new as a version string that does not - contain them, and "dev" is replaced with an '@' so that it sorts lower than - than any other pre-release tag. - """ - parts = [] - for part in _parse_version_parts(s.lower()): - if part.startswith('*'): - if part<'*final': # remove '-' before a prerelease tag - while parts and parts[-1]=='*final-': parts.pop() - # remove trailing zeros from each series of numeric parts - while parts and parts[-1]=='00000000': - parts.pop() - parts.append(part) - return tuple(parts) - -class EntryPoint(object): - """Object representing an advertised importable object""" - - def __init__(self, name, module_name, attrs=(), extras=(), dist=None): - if not MODULE(module_name): - raise ValueError("Invalid module name", module_name) - self.name = name - self.module_name = module_name - self.attrs = tuple(attrs) - self.extras = Requirement.parse(("x[%s]" % ','.join(extras))).extras - self.dist = dist - - def __str__(self): - s = "%s = %s" % (self.name, self.module_name) - if self.attrs: - s += ':' + '.'.join(self.attrs) - if self.extras: - s += ' [%s]' % ','.join(self.extras) - return s - - def __repr__(self): - return "EntryPoint.parse(%r)" % str(self) - - def load(self, require=True, env=None, installer=None): - if require: self.require(env, installer) - entry = __import__(self.module_name, globals(),globals(), ['__name__']) - for attr in self.attrs: - try: - entry = getattr(entry,attr) - except AttributeError: - raise ImportError("%r has no %r attribute" % (entry,attr)) - return entry - - def require(self, env=None, installer=None): - if self.extras and not self.dist: - raise UnknownExtra("Can't require() without a distribution", self) - map(working_set.add, - working_set.resolve(self.dist.requires(self.extras),env,installer)) - - - - #@classmethod - def parse(cls, src, dist=None): - """Parse a single entry point from string `src` - - Entry point syntax follows the form:: - - name = some.module:some.attr [extra1,extra2] - - The entry name and module name are required, but the ``:attrs`` and - ``[extras]`` parts are optional - """ - try: - attrs = extras = () - name,value = src.split('=',1) - if '[' in value: - value,extras = value.split('[',1) - req = Requirement.parse("x["+extras) - if req.specs: raise ValueError - extras = req.extras - if ':' in value: - value,attrs = value.split(':',1) - if not MODULE(attrs.rstrip()): - raise ValueError - attrs = attrs.rstrip().split('.') - except ValueError: - raise ValueError( - "EntryPoint must be in 'name=module:attrs [extras]' format", - src - ) - else: - return cls(name.strip(), value.strip(), attrs, extras, dist) - - parse = classmethod(parse) - - - - - - - - - #@classmethod - def parse_group(cls, group, lines, dist=None): - """Parse an entry point group""" - if not MODULE(group): - raise ValueError("Invalid group name", group) - this = {} - for line in yield_lines(lines): - ep = cls.parse(line, dist) - if ep.name in this: - raise ValueError("Duplicate entry point", group, ep.name) - this[ep.name]=ep - return this - - parse_group = classmethod(parse_group) - - #@classmethod - def parse_map(cls, data, dist=None): - """Parse a map of entry point groups""" - if isinstance(data,dict): - data = data.items() - else: - data = split_sections(data) - maps = {} - for group, lines in data: - if group is None: - if not lines: - continue - raise ValueError("Entry points must be listed in groups") - group = group.strip() - if group in maps: - raise ValueError("Duplicate group name", group) - maps[group] = cls.parse_group(group, lines, dist) - return maps - - parse_map = classmethod(parse_map) - - - - - - -class Distribution(object): - """Wrap an actual or potential sys.path entry w/metadata""" - def __init__(self, - location=None, metadata=None, project_name=None, version=None, - py_version=PY_MAJOR, platform=None, precedence = EGG_DIST - ): - self.project_name = safe_name(project_name or 'Unknown') - if version is not None: - self._version = safe_version(version) - self.py_version = py_version - self.platform = platform - self.location = location - self.precedence = precedence - self._provider = metadata or empty_provider - - #@classmethod - def from_location(cls,location,basename,metadata=None,**kw): - project_name, version, py_version, platform = [None]*4 - basename, ext = os.path.splitext(basename) - if ext.lower() in (".egg",".egg-info"): - match = EGG_NAME(basename) - if match: - project_name, version, py_version, platform = match.group( - 'name','ver','pyver','plat' - ) - return cls( - location, metadata, project_name=project_name, version=version, - py_version=py_version, platform=platform, **kw - ) - from_location = classmethod(from_location) - - hashcmp = property( - lambda self: ( - getattr(self,'parsed_version',()), self.precedence, self.key, - -len(self.location or ''), self.location, self.py_version, - self.platform - ) - ) - def __cmp__(self, other): return cmp(self.hashcmp, other) - def __hash__(self): return hash(self.hashcmp) - - # These properties have to be lazy so that we don't have to load any - # metadata until/unless it's actually needed. (i.e., some distributions - # may not know their name or version without loading PKG-INFO) - - #@property - def key(self): - try: - return self._key - except AttributeError: - self._key = key = self.project_name.lower() - return key - key = property(key) - - #@property - def parsed_version(self): - try: - return self._parsed_version - except AttributeError: - self._parsed_version = pv = parse_version(self.version) - return pv - - parsed_version = property(parsed_version) - - #@property - def version(self): - try: - return self._version - except AttributeError: - for line in self._get_metadata('PKG-INFO'): - if line.lower().startswith('version:'): - self._version = safe_version(line.split(':',1)[1].strip()) - return self._version - else: - raise ValueError( - "Missing 'Version:' header and/or PKG-INFO file", self - ) - version = property(version) - - - - - #@property - def _dep_map(self): - try: - return self.__dep_map - except AttributeError: - dm = self.__dep_map = {None: []} - for name in 'requires.txt', 'depends.txt': - for extra,reqs in split_sections(self._get_metadata(name)): - if extra: extra = safe_extra(extra) - dm.setdefault(extra,[]).extend(parse_requirements(reqs)) - return dm - _dep_map = property(_dep_map) - - def requires(self,extras=()): - """List of Requirements needed for this distro if `extras` are used""" - dm = self._dep_map - deps = [] - deps.extend(dm.get(None,())) - for ext in extras: - try: - deps.extend(dm[safe_extra(ext)]) - except KeyError: - raise UnknownExtra( - "%s has no such extra feature %r" % (self, ext) - ) - return deps - - def _get_metadata(self,name): - if self.has_metadata(name): - for line in self.get_metadata_lines(name): - yield line - - def activate(self,path=None): - """Ensure distribution is importable on `path` (default=sys.path)""" - if path is None: path = sys.path - self.insert_on(path) - if path is sys.path: - fixup_namespace_packages(self.location) - map(declare_namespace, self._get_metadata('namespace_packages.txt')) - - - def egg_name(self): - """Return what this distribution's standard .egg filename should be""" - filename = "%s-%s-py%s" % ( - to_filename(self.project_name), to_filename(self.version), - self.py_version or PY_MAJOR - ) - - if self.platform: - filename += '-'+self.platform - return filename - - def __repr__(self): - if self.location: - return "%s (%s)" % (self,self.location) - else: - return str(self) - - def __str__(self): - try: version = getattr(self,'version',None) - except ValueError: version = None - version = version or "[unknown version]" - return "%s %s" % (self.project_name,version) - - def __getattr__(self,attr): - """Delegate all unrecognized public attributes to .metadata provider""" - if attr.startswith('_'): - raise AttributeError,attr - return getattr(self._provider, attr) - - #@classmethod - def from_filename(cls,filename,metadata=None, **kw): - return cls.from_location( - _normalize_cached(filename), os.path.basename(filename), metadata, - **kw - ) - from_filename = classmethod(from_filename) - - def as_requirement(self): - """Return a ``Requirement`` that matches this distribution exactly""" - return Requirement.parse('%s==%s' % (self.project_name, self.version)) - - def load_entry_point(self, group, name): - """Return the `name` entry point of `group` or raise ImportError""" - ep = self.get_entry_info(group,name) - if ep is None: - raise ImportError("Entry point %r not found" % ((group,name),)) - return ep.load() - - def get_entry_map(self, group=None): - """Return the entry point map for `group`, or the full entry map""" - try: - ep_map = self._ep_map - except AttributeError: - ep_map = self._ep_map = EntryPoint.parse_map( - self._get_metadata('entry_points.txt'), self - ) - if group is not None: - return ep_map.get(group,{}) - return ep_map - - def get_entry_info(self, group, name): - """Return the EntryPoint object for `group`+`name`, or ``None``""" - return self.get_entry_map(group).get(name) - - - - - - - - - - - - - - - - - - - - def insert_on(self, path, loc = None): - """Insert self.location in path before its nearest parent directory""" - - loc = loc or self.location - if not loc: - return - - nloc = _normalize_cached(loc) - bdir = os.path.dirname(nloc) - npath= [(p and _normalize_cached(p) or p) for p in path] - - bp = None - for p, item in enumerate(npath): - if item==nloc: - break - elif item==bdir and self.precedence==EGG_DIST: - # if it's an .egg, give it precedence over its directory - if path is sys.path: - self.check_version_conflict() - path.insert(p, loc) - npath.insert(p, nloc) - break - else: - if path is sys.path: - self.check_version_conflict() - path.append(loc) - return - - # p is the spot where we found or inserted loc; now remove duplicates - while 1: - try: - np = npath.index(nloc, p+1) - except ValueError: - break - else: - del npath[np], path[np] - p = np # ha! - - return - - - def check_version_conflict(self): - if self.key=='setuptools': - return # ignore the inevitable setuptools self-conflicts :( - - nsp = dict.fromkeys(self._get_metadata('namespace_packages.txt')) - loc = normalize_path(self.location) - for modname in self._get_metadata('top_level.txt'): - if (modname not in sys.modules or modname in nsp - or modname in _namespace_packages - ): - continue - - fn = getattr(sys.modules[modname], '__file__', None) - if fn and (normalize_path(fn).startswith(loc) or fn.startswith(loc)): - continue - issue_warning( - "Module %s was already imported from %s, but %s is being added" - " to sys.path" % (modname, fn, self.location), - ) - - def has_version(self): - try: - self.version - except ValueError: - issue_warning("Unbuilt egg for "+repr(self)) - return False - return True - - def clone(self,**kw): - """Copy this distribution, substituting in any changed keyword args""" - for attr in ( - 'project_name', 'version', 'py_version', 'platform', 'location', - 'precedence' - ): - kw.setdefault(attr, getattr(self,attr,None)) - kw.setdefault('metadata', self._provider) - return self.__class__(**kw) - - - - - #@property - def extras(self): - return [dep for dep in self._dep_map if dep] - extras = property(extras) - - -def issue_warning(*args,**kw): - level = 1 - g = globals() - try: - # find the first stack frame that is *not* code in - # the pkg_resources module, to use for the warning - while sys._getframe(level).f_globals is g: - level += 1 - except ValueError: - pass - from warnings import warn - warn(stacklevel = level+1, *args, **kw) - - - - - - - - - - - - - - - - - - - - - - - -def parse_requirements(strs): - """Yield ``Requirement`` objects for each specification in `strs` - - `strs` must be an instance of ``basestring``, or a (possibly-nested) - iterable thereof. - """ - # create a steppable iterator, so we can handle \-continuations - lines = iter(yield_lines(strs)) - - def scan_list(ITEM,TERMINATOR,line,p,groups,item_name): - - items = [] - - while not TERMINATOR(line,p): - if CONTINUE(line,p): - try: - line = lines.next(); p = 0 - except StopIteration: - raise ValueError( - "\\ must not appear on the last nonblank line" - ) - - match = ITEM(line,p) - if not match: - raise ValueError("Expected "+item_name+" in",line,"at",line[p:]) - - items.append(match.group(*groups)) - p = match.end() - - match = COMMA(line,p) - if match: - p = match.end() # skip the comma - elif not TERMINATOR(line,p): - raise ValueError( - "Expected ',' or end-of-list in",line,"at",line[p:] - ) - - match = TERMINATOR(line,p) - if match: p = match.end() # skip the terminator, if any - return line, p, items - - for line in lines: - match = DISTRO(line) - if not match: - raise ValueError("Missing distribution spec", line) - project_name = match.group(1) - p = match.end() - extras = [] - - match = OBRACKET(line,p) - if match: - p = match.end() - line, p, extras = scan_list( - DISTRO, CBRACKET, line, p, (1,), "'extra' name" - ) - - line, p, specs = scan_list(VERSION,LINE_END,line,p,(1,2),"version spec") - specs = [(op,safe_version(val)) for op,val in specs] - yield Requirement(project_name, specs, extras) - - -def _sort_dists(dists): - tmp = [(dist.hashcmp,dist) for dist in dists] - tmp.sort() - dists[::-1] = [d for hc,d in tmp] - - - - - - - - - - - - - - - - - -class Requirement: - def __init__(self, project_name, specs, extras): - """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" - self.unsafe_name, project_name = project_name, safe_name(project_name) - self.project_name, self.key = project_name, project_name.lower() - index = [(parse_version(v),state_machine[op],op,v) for op,v in specs] - index.sort() - self.specs = [(op,ver) for parsed,trans,op,ver in index] - self.index, self.extras = index, tuple(map(safe_extra,extras)) - self.hashCmp = ( - self.key, tuple([(op,parsed) for parsed,trans,op,ver in index]), - frozenset(self.extras) - ) - self.__hash = hash(self.hashCmp) - - def __str__(self): - specs = ','.join([''.join(s) for s in self.specs]) - extras = ','.join(self.extras) - if extras: extras = '[%s]' % extras - return '%s%s%s' % (self.project_name, extras, specs) - - def __eq__(self,other): - return isinstance(other,Requirement) and self.hashCmp==other.hashCmp - - def __contains__(self,item): - if isinstance(item,Distribution): - if item.key != self.key: return False - if self.index: item = item.parsed_version # only get if we need it - elif isinstance(item,basestring): - item = parse_version(item) - last = None - for parsed,trans,op,ver in self.index: - action = trans[cmp(item,parsed)] - if action=='F': return False - elif action=='T': return True - elif action=='+': last = True - elif action=='-' or last is None: last = False - if last is None: last = True # no rules encountered - return last - - - def __hash__(self): - return self.__hash - - def __repr__(self): return "Requirement.parse(%r)" % str(self) - - #@staticmethod - def parse(s): - reqs = list(parse_requirements(s)) - if reqs: - if len(reqs)==1: - return reqs[0] - raise ValueError("Expected only one requirement", s) - raise ValueError("No requirements found", s) - - parse = staticmethod(parse) - -state_machine = { - # =>< - '<' : '--T', - '<=': 'T-T', - '>' : 'F+F', - '>=': 'T+F', - '==': 'T..', - '!=': 'F++', -} - - -def _get_mro(cls): - """Get an mro for a type or classic class""" - if not isinstance(cls,type): - class cls(cls,object): pass - return cls.__mro__[1:] - return cls.__mro__ - -def _find_adapter(registry, ob): - """Return an adapter factory for `ob` from `registry`""" - for t in _get_mro(getattr(ob, '__class__', type(ob))): - if t in registry: - return registry[t] - - -def ensure_directory(path): - """Ensure that the parent directory of `path` exists""" - dirname = os.path.dirname(path) - if not os.path.isdir(dirname): - os.makedirs(dirname) - -def split_sections(s): - """Split a string or iterable thereof into (section,content) pairs - - Each ``section`` is a stripped version of the section header ("[section]") - and each ``content`` is a list of stripped lines excluding blank lines and - comment-only lines. If there are any such lines before the first section - header, they're returned in a first ``section`` of ``None``. - """ - section = None - content = [] - for line in yield_lines(s): - if line.startswith("["): - if line.endswith("]"): - if section or content: - yield section, content - section = line[1:-1].strip() - content = [] - else: - raise ValueError("Invalid section heading", line) - else: - content.append(line) - - # wrap up last segment - yield section, content - -def _mkstemp(*args,**kw): - from tempfile import mkstemp - old_open = os.open - try: - os.open = os_open # temporarily bypass sandboxing - return mkstemp(*args,**kw) - finally: - os.open = old_open # and then put it back - - -# Set up global resource manager (deliberately not state-saved) -_manager = ResourceManager() -def _initialize(g): - for name in dir(_manager): - if not name.startswith('_'): - g[name] = getattr(_manager, name) -_initialize(globals()) - -# Prepare the master working set and make the ``require()`` API available -_declare_state('object', working_set = WorkingSet()) -try: - # Does the main program list any requirements? - from __main__ import __requires__ -except ImportError: - pass # No: just use the default working set based on sys.path -else: - # Yes: ensure the requirements are met, by prefixing sys.path if necessary - try: - working_set.require(__requires__) - except VersionConflict: # try it without defaults already on sys.path - working_set = WorkingSet([]) # by starting with an empty path - for dist in working_set.resolve( - parse_requirements(__requires__), Environment() - ): - working_set.add(dist) - for entry in sys.path: # add any missing entries from sys.path - if entry not in working_set.entries: - working_set.add_entry(entry) - sys.path[:] = working_set.entries # then copy back to sys.path - -require = working_set.require -iter_entry_points = working_set.iter_entry_points -add_activation_listener = working_set.subscribe -run_script = working_set.run_script -run_main = run_script # backward compatibility -# Activate all distributions already on sys.path, and ensure that -# all distributions added to the working set in the future (e.g. by -# calling ``require()``) will get activated as well. -add_activation_listener(lambda dist: dist.activate()) -working_set.entries=[]; map(working_set.add_entry,sys.path) # match order - diff --git a/run.sh b/run.sh index 6744f72f675..ebc66407c5e 100755 --- a/run.sh +++ b/run.sh @@ -2,14 +2,6 @@ cd `dirname $0` -# If there is a .venv/ directory, assume it contains a virtualenv that we -# should run this instance in. -if [ -d .venv ]; -then - printf "Activating virtualenv at %s/.venv\n" $(pwd) - . .venv/bin/activate -fi - # If there is a file that defines a shell environment specific to this # instance of Galaxy, source the file. if [ -z "$GALAXY_LOCAL_ENV_FILE" ]; @@ -27,6 +19,14 @@ python ./scripts/check_python.py ./scripts/common_startup.sh +# If there is a .venv/ directory, assume it contains a virtualenv that we +# should run this instance in. +if [ -d .venv ]; +then + printf "Activating virtualenv at %s/.venv\n" $(pwd) + . .venv/bin/activate +fi + if [ -n "$GALAXY_UNIVERSE_CONFIG_DIR" ]; then python ./scripts/build_universe_config.py "$GALAXY_UNIVERSE_CONFIG_DIR" fi diff --git a/scripts/check_eggs.py b/scripts/check_eggs.py deleted file mode 100644 index 7655eb964cb..00000000000 --- a/scripts/check_eggs.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -""" -Compares local dependency eggs to those in eggs.ini, displaying a warning if -any are out of date. - -usage: check_eggs.py [options] -""" - -import os, sys, logging -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=None ) -parser.add_option( '-q', '--quiet', dest='quiet', action="store_true", help='Quiet (no output, only set return code)', default=False ) -( options, args ) = parser.parse_args() - - -config_set = True -config = options.config -if config is None: - config_set = False - for name in ['config/galaxy.ini', 'universe_wsgi.ini', 'config/galaxy.ini.sample']: - if os.path.exists(name): - config = name - break - -if not os.path.exists( config ): - print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], config ) - sys.exit( 1 ) - -root = logging.getLogger() -root.setLevel( 10 ) -root.addHandler( logging.StreamHandler( sys.stdout ) ) - -config_arg = '' -if config_set: - config_arg = '-c %s' % config - -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) -sys.path.insert( 1, lib ) - -from galaxy.eggs import Crate - -c = Crate( config ) -if c.config_missing: - if not options.quiet: - print "Some of your Galaxy eggs are out of date. Please update them" - print "by running:" - print " python scripts/fetch_eggs.py %s" % config_arg - sys.exit( 1 ) -sys.exit( 0 ) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 34c7e261500..e6bfbcdbeab 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -1,11 +1,14 @@ #!/bin/bash # explicitly attempt to fetch eggs before running -FETCH_EGGS=1 +FETCH_WHEELS=1 +SET_VENV=1 COPY_SAMPLE_FILES=1 for arg in "$@"; do - [ "$arg" = "--skip-eggs" ] && FETCH_EGGS=0 - [ "$arg" = "--stop-daemon" ] && FETCH_EGGS=0 + [ "$arg" = "--skip-eggs" ] && FETCH_WHEELS=0 + [ "$arg" = "--skip-wheels" ] && FETCH_WHEELS=0 + [ "$arg" = "--skip-venv" ] && SET_VENV=0 + [ "$arg" = "--stop-daemon" ] && FETCH_WHEELS=0 [ "$arg" = "--skip-samples" ] && COPY_SAMPLE_FILES=0 done @@ -23,6 +26,10 @@ SAMPLES=" static/welcome.html.sample " +RMFILES=" + lib/pkg_resources.pyc +" + if [ $COPY_SAMPLE_FILES -eq 1 ]; then # Create any missing config/location files for sample in $SAMPLES; do @@ -34,6 +41,11 @@ if [ $COPY_SAMPLE_FILES -eq 1 ]; then done fi +# remove problematic cached files +for rmfile in $RMFILES; do + [ -f $rmfile ] && rm -f $rmfile +done + : ${GALAXY_CONFIG_FILE:=config/galaxy.ini} if [ ! -f $GALAXY_CONFIG_FILE ]; then GALAXY_CONFIG_FILE=universe_wsgi.ini @@ -42,16 +54,41 @@ if [ ! -f $GALAXY_CONFIG_FILE ]; then GALAXY_CONFIG_FILE=config/galaxy.ini.sample fi -if [ $FETCH_EGGS -eq 1 ]; then - python ./scripts/check_eggs.py -q -c $GALAXY_CONFIG_FILE - if [ $? -ne 0 ]; then - echo "Some eggs are out of date, attempting to fetch..." - python ./scripts/fetch_eggs.py -c $GALAXY_CONFIG_FILE - if [ $? -eq 0 ]; then - echo "Fetch successful." +if [ $SET_VENV -eq 1 ]; then + # If not already in a virtualenv and .venv does not exist, attempt to create it. + if [ -z "$VIRTUAL_ENV" -a ! -d .venv ] + then + vvers=13.1.0 + vurl="https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${vvers}.tar.gz" + if command -v virtualenv >/dev/null; then + virtualenv .venv else - echo "Fetch failed." - exit 1 + vtmp=`mktemp -d -t galaxy-virtualenv-XXXXXX` + vsrc="$vtmp/`basename $vurl`" + ( if command -v curl >/dev/null; then + curl --silent -o $vsrc $vurl + elif command -v wget >/dev/null; then + wget --quiet -O $vsrc $vurl + else + python -c "import urllib; urllib.urlretrieve('$vurl', '$vsrc')" + fi ) && + tar zxf $vsrc -C $vtmp && + python $vtmp/virtualenv-$vvers/virtualenv.py .venv && + rm -rf $vtmp fi fi -fi + + # If there is a .venv/ directory, assume it contains a virtualenv that we + # should run this instance in. + if [ -d .venv ]; + then + printf "Activating virtualenv at %s/.venv\n" $(pwd) + . .venv/bin/activate + fi +fi + + +if [ $FETCH_WHEELS -eq 1 ]; then + PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.requirements('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin + #pip install -r requirements.txt --index-url https://wheels.galaxyproject.org/ +fi diff --git a/scripts/dist-scramble.py b/scripts/dist-scramble.py deleted file mode 100644 index 04f396be393..00000000000 --- a/scripts/dist-scramble.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Scrambles eggs for distribution on multiple platforms. - -(from https://wiki.galaxyproject.org/Admin/Config/Eggs ) -This is mostly designed for use by Galaxy Developers at Penn State who are -building eggs for distribution via the Galaxy Eggs distribution site. -dist-scramble.py uses the dist-eggs.ini config file to determine what platforms -to build for, and which hosts to build on. - -dist-scramble.py works the same way as scramble.py: :: - -% python scripts/dist-scramble.py galaxy_egg - -Called with only the egg argument, dist-scramble.py will build for all the -platforms under the all group in its config file (for platform-specific eggs) -or the noplatform group (for platform-inspecific eggs). The [[hosts]|section -contains information about which hosts will be used for building on each desired -platform. If you don't want to build for all the platforms listed under the all -group, you can add a platform argument (any lvalue in the [hosts]] or [groups] -section is valid): :: - -% python scripts/dist-scramble.py galaxy_egg linux - -The platform argument is ignored for platform-inspecific eggs. An assumption is -made that your Galaxy distribution is located at the same place on all of the -hosts on which you're building (i.e. via a network filesystem). - -Once dist-scramble.py finishes, it will output a list of platforms on which it -failed to scramble the egg. Successful eggs will be put in a new dist-eggs -subdirectory of your Galaxy distribution. These eggs can then be copied to your -distribution site. -""" - -import os, sys, logging -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( '-e', '--egg-name', dest='egg_name', help='Egg name (as defined in eggs.ini) to scramble (required)' ) -parser.add_option( '-p', '--platform', dest='platform', help='Scramble for a specific platform (by default, eggs are scrambled for all platforms, see dist-eggs.ini for platform names)' ) -( options, args ) = parser.parse_args() - -root = logging.getLogger() -root.setLevel( 10 ) -root.addHandler( logging.StreamHandler( sys.stdout ) ) - -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) -sys.path.insert( 1, lib ) - -from galaxy.eggs.dist import DistScrambleCrate, ScrambleFailure -from galaxy.eggs import EggNotFetchable - -if not options.egg_name: - print "ERROR: You must specify an egg to scramble (-e)" - parser.print_help() - sys.exit( 1 ) - -if options.platform: - c = DistScrambleCrate( None, options.platform ) -else: - c = DistScrambleCrate( None ) - -try: - eggs = c[options.egg_name] -except: - print "ERROR: %s not in eggs.ini" % options.egg_name - sys.exit( 1 ) -failed = [] -for egg in eggs: - try: - for dependency in egg.dependencies: - print "Checking %s on %s dependency: %s" % ( egg.name, egg.platform, dependency ) - # this could be in a better data structure... - dep = filter( lambda x: x.platform == egg.platform, c[dependency] )[0] - if not os.path.exists( dep.distribution.location ): - dep.fetch( dep.distribution.as_requirement() ) - except EggNotFetchable, e: - degg = e.eggs[0] - print "%s build dependency %s %s %s couldn't be" % ( egg.name, degg.name, degg.version, degg.platform ) - print "downloaded automatically. There isn't really a graceful" - print "way to handle this when dist-scrambling." - failed.append( egg.platform ) - continue - try: - egg.scramble() - except ScrambleFailure: - failed.append( egg.platform ) -if len( failed ): - print "" - print "Scramble failed to build eggs on the following platforms (more details" - print "can be found by reviewing the output above):" - print "\n".join( failed ) diff --git a/scripts/fetch_eggs.py b/scripts/fetch_eggs.py deleted file mode 100755 index 20f9c8dfcd2..00000000000 --- a/scripts/fetch_eggs.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -Connects to the Galaxy Eggs distribution site and downloads any eggs needed. - -If eggs for your platform are unavailable, fetch_eggs.py will direct you to run -scramble.py. -""" - -import os, sys, logging -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=None ) -parser.add_option( '-e', '--egg-name', dest='egg_name', help='Egg name (as defined in eggs.ini) to fetch, or "all" for all eggs, even those not needed by your configuration' ) -parser.add_option( '-p', '--platform', dest='platform', help='Fetch for a specific platform (by default, eggs are fetched for *this* platform' ) -( options, args ) = parser.parse_args() - -config_set = True -config = options.config -if config is None: - config_set = False - for name in ['config/galaxy.ini', 'universe_wsgi.ini', 'config/galaxy.ini.sample']: - if os.path.exists(name): - config = name - break - -if not os.path.exists( config ): - print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], config ) - sys.exit( 1 ) - -root = logging.getLogger() -root.setLevel( 10 ) -root.addHandler( logging.StreamHandler( sys.stdout ) ) - -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) -sys.path.insert(1, lib) - -from galaxy.eggs import Crate, EggNotFetchable -import pkg_resources - -if options.platform: - c = Crate( config, platform = options.platform ) -else: - c = Crate( config ) -try: - if not options.egg_name: - c.resolve() # Only fetch eggs required by the config - elif options.egg_name == 'all': - c.resolve( all=True ) # Fetch everything - else: - # Fetch a specific egg - name = options.egg_name - try: - egg = c[name] - except: - print "error: %s not in eggs.ini" % name - sys.exit( 1 ) - dist = egg.resolve()[0] - print "%s %s is installed at %s" % ( dist.project_name, dist.version, dist.location ) -except EggNotFetchable, e: - config_arg = '' - if config_set: - config_arg = '-c %s ' % config - try: - assert options.egg_name != 'all' - egg = e.eggs[0] - print "%s %s couldn't be downloaded automatically. You can try" % ( egg.name, egg.version ) - print "building it by hand with:" - print " python scripts/scramble.py %s-e %s" % ( config_arg, egg.name ) - except ( AssertionError, IndexError ): - print "One or more of the python eggs necessary to run Galaxy couldn't be" - print "downloaded automatically. You can try building them by hand (all" - print "at once) with:" - print " python scripts/scramble.py" - print "Or individually:" - for egg in e.eggs: - print " python scripts/scramble.py %s-e %s" % ( config_arg, egg.name ) - sys.exit( 1 ) -sys.exit( 0 ) diff --git a/scripts/make_egg_packager.py b/scripts/make_egg_packager.py deleted file mode 100644 index e9d2352b7c7..00000000000 --- a/scripts/make_egg_packager.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python - -import os, sys, logging, shutil -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default='config/galaxy.ini' ) -parser.add_option( '-p', '--platform', dest='platform', help='Fetch for a specific platform (by default, eggs are fetched for *this* platform' ) -( options, args ) = parser.parse_args() - -if not os.path.exists( options.config ): - print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], options.config ) - sys.exit( 1 ) - -root = logging.getLogger() -root.setLevel( 10 ) -root.addHandler( logging.StreamHandler( sys.stdout ) ) - -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) -sys.path.insert( 1, lib ) - -from galaxy.eggs import Crate, EggNotFetchable, py -import pkg_resources - -try: - assert options.platform - platform = options.platform - c = Crate( options.config, platform = platform ) - print "Platform forced to '%s'" % platform -except: - platform = '-'.join( ( py, pkg_resources.get_platform() ) ) - c = Crate( options.config ) - print "Using Python interpreter at %s, Version %s" % ( sys.executable, sys.version ) - print "This platform is '%s'" % platform - print "Override with:" - print " make_egg_packager.py " - -shutil.copy( os.path.join( os.path.dirname( __file__ ), 'egg_packager_template.py' ), 'egg_packager-%s.py' % platform ) - -packager = open( 'egg_packager-%s.py' % platform, 'a' ) -packager.write( "py = '%s'\n" % py ) -packager.write( "url = '%s'\n" % c.repo ) -packager.write( "platform = '%s'\n" % platform ) -packager.write( "dists = [\n" ) - -for egg in c.all_eggs: - if egg.name in c.no_auto: - continue - packager.write( " Distribution( '%s', '%s', '%s', '%s', '%s' ),\n" % ( egg.distribution.egg_name(), egg.distribution.project_name, egg.distribution.version, egg.distribution.py_version, egg.distribution.platform ) ) - -packager.write( """] - -for d in dists: - e = Egg( d ) - if not e.fetch( None ): - failures.append( e ) - -if failures: - print "" - print "Failed:" - for e in failures: - print e.distribution.project_name -else: - create_zip() -clean() -""" ) - -print "Completed packager is 'egg_packager-%s.py'. To" % platform -print "fetch eggs, please copy this file to a system with internet access and run" -print "with:" -print " python egg_packager-%s.py" % platform diff --git a/scripts/scramble.py b/scripts/scramble.py deleted file mode 100644 index ed97a5fe527..00000000000 --- a/scripts/scramble.py +++ /dev/null @@ -1,58 +0,0 @@ -import os, sys, logging -from optparse import OptionParser - -parser = OptionParser() -parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default='config/galaxy.ini' ) -parser.add_option( '-e', '--egg-name', dest='egg_name', help='Egg name (as defined in eggs.ini) to fetch, or "all" for all eggs, even those not needed by your configuration' ) -( options, args ) = parser.parse_args() - -if not os.path.exists( options.config ): - print "Config file does not exist (see 'python %s --help'): %s" % ( sys.argv[0], options.config ) - sys.exit( 1 ) - -root = logging.getLogger() -root.setLevel( 10 ) -root.addHandler( logging.StreamHandler( sys.stdout ) ) - -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) -sys.path.insert( 1, lib ) - -from galaxy.eggs.scramble import ScrambleCrate, ScrambleFailure, EggNotFetchable - -c = ScrambleCrate( options.config ) - -try: - if not options.egg_name: - eggs = c.scramble() - elif options.egg_name == 'all': - c.scramble( all=True ) - else: - # Scramble a specific egg - name = options.egg_name - try: - egg = c[name] - except: - print "error: %s not in eggs.ini" % name - sys.exit( 1 ) - for dependency in egg.dependencies: - config_arg = '' - if options.config != 'config/galaxy.ini': - config_arg = '-c %s' % options.config - print "Checking %s dependency: %s" % ( egg.name, dependency ) - try: - c[dependency].require() - except EggNotFetchable, e: - degg = e.eggs[0] - print "%s build dependency %s %s couldn't be downloaded" % ( egg.name, degg.name, degg.version ) - print "automatically. You can try building it by hand with:" - print " python scripts/scramble.py %s-e %s" % ( config_arg, degg.name ) - sys.exit( 1 ) - egg.scramble() - sys.exit( 0 ) -except ScrambleFailure, e: - if len( e.eggs ) == 1: - raise - else: - print 'Scrambling the following eggs failed:\n ', - print '\n '.join( [ egg.name for egg in e.eggs ] ) - sys.exit( 1 ) diff --git a/scripts/scramble/lib/scramble_lib.py b/scripts/scramble/lib/scramble_lib.py deleted file mode 100644 index 607b865b67b..00000000000 --- a/scripts/scramble/lib/scramble_lib.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Various utilities for scrambling. -""" -import os, sys, errno, re, distutils.util, glob, shutil, subprocess, tarfile, zipfile -from distutils.sysconfig import get_config_var, get_config_vars - -try: - import zlib -except: - raise Exception( 'Cannot import zlib, which must exist to build eggs. If your python interpreter is truly missing it, you will need to recompile or (on supported platforms) download a binary version from python.org.' ) - -def get_tag(): - try: - return open( '.galaxy_tag', 'r' ).read().strip() - except: - return None - -def get_ver(): - try: - return open( '.galaxy_ver', 'r' ).read().strip() - except: - return None - -def get_deps(): - try: - depf = open( '.galaxy_deps', 'r' ) - except: - return [] - c = eggs.Crate( None ) - for dep in depf: - c[dep.strip()].require() - -def clean( extra_dirs=[] ): - for dir in [ 'build', 'dist' ] + extra_dirs: - try: - shutil.rmtree( dir ) - except OSError, e: - if e.errno != errno.ENOENT: - raise - -def apply_patches(): - name = os.path.basename( os.getcwd() ) - for file in glob.glob( os.path.join( patches, name, '*' ) ): - shutil.copy( file, os.path.basename( file ) ) - -def get_archive( base ): - for arctype in ( 'tar.gz', 'tgz', 'zip' ): - archive = '.'.join( ( base, arctype ) ) - if os.path.exists( archive ): - return archive - else: - raise Exception( "%s(): Couldn't find a suitable archive for %s in %s" % ( sys._getframe().f_code.co_name, os.path.basename( base ), archives ) ) - -def compress( path, *files ): - tarcf( path, *files ) - -def uncompress( path ): - if path.endswith( '.zip' ): - unzip( path ) - else: - tarxf( path ) - -def unzip( zipf ): - z = zipfile.ZipFile( zipf ) - try: - for info in z.infolist(): - name = info.filename - mode = (info.external_attr >> 16L) & 0777 - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - if not name: - continue - if name.endswith('/'): - # directory - pkg_resources.ensure_directory(name) - else: - # file - pkg_resources.ensure_directory(name) - data = z.read(info.filename) - f = open(name,'wb') - try: - f.write(data) - finally: - f.close() - del data - try: - if not os.path.islink( name ): - os.chmod(name,mode) - except: - pass - finally: - z.close() - -def tarxf( ball ): - t = tarfile.open( ball, 'r' ) - for fn in t.getnames(): - t.extract( fn ) - t.close() - -def tarcf( ball, *files ): - if ball.endswith( '.gz' ): - t = tarfile.open( ball, 'w:gz' ) - elif ball.endswith( '.bz2' ): - t = tarfile.open( ball, 'w:bz2' ) - else: - t = tarfile.open( ball, 'w' ) - for file in files: - t.add( file ) - t.close() - -def run( cmd, d, txt ): - p = subprocess.Popen( args = cmd, shell = True, cwd = d ) - r = p.wait() - if r != 0: - print '%s(): %s failed' % ( sys._getframe().f_code.co_name, txt ) - sys.exit( 1 ) - -def unpack_dep( source, prepped, builder, args={} ): - if prepped is not None and os.path.exists( prepped ): - print "%s(): Prepared dependency already exists at the following path, remove to force re-prep:" % sys._getframe().f_code.co_name - print " ", prepped - uncompress( prepped ) - else: - print "%s(): Prepared dependency does not exist, preparing now from source:" % sys._getframe().f_code.co_name - print " ", source - uncompress( source ) - builder( prepped, args ) - -def get_solaris_compiler(): - p = subprocess.Popen( '%s -V 2>&1' % get_config_var('CC'), shell = True, stdout = subprocess.PIPE ) - out = p.stdout.read() - p.wait() - if 'Sun C' in out: - return 'cc' - else: - return 'gcc' - -# get galaxy eggs lib -galaxy_lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', '..', '..', 'lib' ) ) -sys.path.insert( 0, galaxy_lib ) -from galaxy import eggs - -# get setuptools -try: - from setuptools import * - import pkg_resources -except ImportError: - from ez_setup import use_setuptools - use_setuptools( download_delay=8, to_dir=os.path.dirname( __file__ ) ) - from setuptools import * - import pkg_resources - -# some constants -root = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..' ) ) -archives = os.path.abspath( os.path.join( root, 'archives' ) ) -patches = os.path.abspath( os.path.join( root, 'patches' ) ) -platform_noucs = pkg_resources.get_platform().rsplit( '-', 1 )[0] diff --git a/scripts/scramble/patches/Cheetah/SetupConfig.py b/scripts/scramble/patches/Cheetah/SetupConfig.py deleted file mode 100644 index 8d5227cc7e9..00000000000 --- a/scripts/scramble/patches/Cheetah/SetupConfig.py +++ /dev/null @@ -1,103 +0,0 @@ -#-------Main Package Settings-----------# -import sys - -name = 'Cheetah' -from cheetah.Version import Version as version -maintainer = "R. Tyler Ballance" -author = "Tavis Rudd" -author_email = "cheetahtemplate-discuss@lists.sf.net" -url = "http://www.cheetahtemplate.org/" -packages = ['Cheetah', - 'Cheetah.Macros', - 'Cheetah.Templates', - 'Cheetah.Tests', - 'Cheetah.Tools', - 'Cheetah.Utils', - ] -classifiers = [line.strip() for line in '''\ - #Development Status :: 4 - Beta - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - Intended Audience :: System Administrators - License :: OSI Approved :: MIT License - Operating System :: OS Independent - Programming Language :: Python - Topic :: Internet :: WWW/HTTP - Topic :: Internet :: WWW/HTTP :: Dynamic Content - Topic :: Internet :: WWW/HTTP :: Site Management - Topic :: Software Development :: Code Generators - Topic :: Software Development :: Libraries :: Python Modules - Topic :: Software Development :: User Interfaces - Topic :: Text Processing'''.splitlines() if not line.strip().startswith('#')] -del line - -package_dir = {'Cheetah':'cheetah'} - -import os -import os.path -from distutils.core import Extension - -ext_modules=[ - Extension("Cheetah._namemapper", - [os.path.join('cheetah', 'c', '_namemapper.c')]), - # Extension("Cheetah._verifytype", - # [os.path.join('cheetah', 'c', '_verifytype.c')]), - # Extension("Cheetah._filters", - # [os.path.join('cheetah', 'c', '_filters.c')]), - # Extension('Cheetah._template', - # [os.path.join('cheetah', 'c', '_template.c')]), - ] - -## Data Files and Scripts -scripts = ['bin/cheetah-compile', - 'bin/cheetah', - ] - -data_files = ['recursive: cheetah *.tmpl *.txt LICENSE README TODO CHANGES',] - -if not os.getenv('CHEETAH_INSTALL_WITHOUT_SETUPTOOLS'): - try: - from setuptools import setup -# install_requires = [ -# "Markdown >= 2.0.1", -# ] - if sys.platform == 'win32': - # use 'entry_points' instead of 'scripts' - del scripts - entry_points = { - 'console_scripts': [ - 'cheetah = Cheetah.CheetahWrapper:_cheetah', - 'cheetah-compile = Cheetah.CheetahWrapper:_cheetah_compile', - ] - } - except ImportError: - print 'Not using setuptools, so we cannot install the Markdown dependency' - - -description = "Cheetah is a template engine and code generation tool." - -long_description = '''Cheetah is an open source template engine and code generation tool. - -It can be used standalone or combined with other tools and frameworks. Web -development is its principal use, but Cheetah is very flexible and is also being -used to generate C++ game code, Java, sql, form emails and even Python code. - -Documentation -================================================================================ -For a high-level introduction to Cheetah please refer to the User\'s Guide -at http://www.cheetahtemplate.org/learn.html - -Mailing list -================================================================================ -cheetahtemplate-discuss@lists.sourceforge.net -Subscribe at http://lists.sourceforge.net/lists/listinfo/cheetahtemplate-discuss - -Credits -================================================================================ -http://www.cheetahtemplate.org/credits.html - -Recent Changes -================================================================================ -See http://www.cheetahtemplate.org/CHANGES.txt for full details - -''' diff --git a/scripts/scramble/patches/MySQL_python/site.cfg b/scripts/scramble/patches/MySQL_python/site.cfg deleted file mode 100644 index 8be48b7606d..00000000000 --- a/scripts/scramble/patches/MySQL_python/site.cfg +++ /dev/null @@ -1,18 +0,0 @@ -[options] -# embedded: link against the embedded server library -# threadsafe: use the threadsafe client -# static: link against a static library (probably required for embedded) - -embedded = False -threadsafe = True -static = True - -# The path to mysql_config. -# Only use this if mysql_config is not on your PATH, or you have some weird -# setup that requires it. -mysql_config = mysql/bin/mysql_config - -# The Windows registry key for MySQL. -# This has to be set for Windows builds to work. -# Only change this if you have a different version. -registry_key = SOFTWARE\MySQL AB\MySQL Server 5.0 diff --git a/scripts/scramble/patches/PasteScript/setup.py b/scripts/scramble/patches/PasteScript/setup.py deleted file mode 100644 index 9907f525328..00000000000 --- a/scripts/scramble/patches/PasteScript/setup.py +++ /dev/null @@ -1,145 +0,0 @@ -import ez_setup -ez_setup.use_setuptools() -from setuptools import setup, find_packages -import re, os - -version = '1.7.3' - -news = os.path.join(os.path.dirname(__file__), 'docs', 'news.txt') -news = open(news).read() -parts = re.split(r'([0-9\.]+)\s*\n\r?-+\n\r?', news) -found_news = '' -for i in range(len(parts)-1): - if parts[i] == version: - found_news = parts[i+i] - break -if not found_news: - print 'Warning: no news for this version found' - -long_description="""\ -This is a pluggable command-line tool. - -It includes some built-in features; - -* Create file layouts for packages. For instance, ``paste create - --template=basic_package MyPackage`` will create a `setuptools - `_-ready - file layout. - -* Serving up web applications, with configuration based on - `paste.deploy `_. - -The latest version is available in a `Subversion repository -`_. - -For the latest changes see the `news file -`_. -""" - -if found_news: - title = 'Changes in %s' % version - long_description += "\n%s\n%s\n" % (title, '-'*len(title)) - long_description += found_news - -setup( - name="PasteScript", - version=version, - description="A pluggable command-line frontend, including commands to setup package file layouts", - long_description=long_description, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Internet :: WWW/HTTP :: Dynamic Content", - "Topic :: Software Development :: Libraries :: Python Modules", - "Framework :: Paste", - ], - keywords='web wsgi setuptools framework command-line setup', - author="Ian Bicking", - author_email="ianb@colorstudy.com", - url="http://pythonpaste.org/script/", - namespace_packages=['paste'], - license='MIT', - packages=find_packages(exclude='tests'), - package_data={ - 'paste.script': ['paster-templates/basic_package/setup.*', - 'paster-templates/basic_package/tests/*.py', - # @@: docs/ doesn't have any files :( - 'paster-templates/basic_package/+package+/*.py'], - }, - zip_safe=False, - scripts=['scripts/paster'], - extras_require={ - 'Templating': [], - 'Cheetah': ['Cheetah'], - 'Config': ['PasteDeploy'], - 'WSGIUtils': ['WSGIUtils'], - 'Flup': ['Flup'], - # the Paste feature means the complete set of features; - # (other features are truly optional) - 'Paste': ['PasteDeploy', 'Cheetah'], - }, - entry_points=""" - [paste.global_paster_command] - help=paste.script.help:HelpCommand - create=paste.script.create_distro:CreateDistroCommand [Templating] - serve=paste.script.serve:ServeCommand [Config] - request=paste.script.request:RequestCommand [Config] - post=paste.script.request:RequestCommand [Config] - exe=paste.script.exe:ExeCommand - points=paste.script.entrypoints:EntryPointCommand - make-config=paste.script.appinstall:MakeConfigCommand - setup-app=paste.script.appinstall:SetupCommand - - [paste.paster_command] - grep = paste.script.grep:GrepCommand - - [paste.paster_create_template] - basic_package=paste.script.templates:BasicPackage - - [paste.server_runner] - wsgiutils=paste.script.wsgiutils_server:run_server [WSGIUtils] - flup_ajp_thread=paste.script.flup_server:run_ajp_thread [Flup] - flup_ajp_fork=paste.script.flup_server:run_ajp_fork [Flup] - flup_fcgi_thread=paste.script.flup_server:run_fcgi_thread [Flup] - flup_fcgi_fork=paste.script.flup_server:run_fcgi_fork [Flup] - flup_scgi_thread=paste.script.flup_server:run_scgi_thread [Flup] - flup_scgi_fork=paste.script.flup_server:run_scgi_fork [Flup] - cgi=paste.script.cgi_server:paste_run_cgi - cherrypy=paste.script.cherrypy_server:cpwsgi_server - twisted=paste.script.twisted_web2_server:run_twisted - - [paste.app_factory] - test=paste.script.testapp:make_test_application - - [paste.entry_point_description] - paste.entry_point_description = paste.script.epdesc:MetaEntryPointDescription - paste.paster_create_template = paste.script.epdesc:CreateTemplateDescription - paste.paster_command = paste.script.epdesc:PasterCommandDescription - paste.global_paster_command = paste.script.epdesc:GlobalPasterCommandDescription - paste.app_install = paste.script.epdesc:AppInstallDescription - - # These aren't part of Paste Script particularly, but - # we'll document them here - console_scripts = paste.script.epdesc:ConsoleScriptsDescription - # @@: Need non-console scripts... - distutils.commands = paste.script.epdesc:DistutilsCommandsDescription - distutils.setup_keywords = paste.script.epdesc:SetupKeywordsDescription - egg_info.writers = paste.script.epdesc:EggInfoWriters - # @@: Not sure what this does: - #setuptools.file_finders = paste.script.epdesc:SetuptoolsFileFinders - - [console_scripts] - paster=paste.script.command:run - - [distutils.setup_keywords] - paster_plugins = setuptools.dist:assert_string_list - - [egg_info.writers] - paster_plugins.txt = setuptools.command.egg_info:write_arg - """, - install_requires=[ - ], - ) diff --git a/scripts/scramble/patches/python_daemon/setup.py b/scripts/scramble/patches/python_daemon/setup.py deleted file mode 100644 index a5e80b525c4..00000000000 --- a/scripts/scramble/patches/python_daemon/setup.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- - -# setup.py -# Part of python-daemon, an implementation of PEP 3143. -# -# Copyright © 2008–2010 Ben Finney -# Copyright © 2008 Robert Niederreiter, Jens Klein -# -# This is free software: you may copy, modify, and/or distribute this work -# under the terms of the Python Software Foundation License, version 2 or -# later as published by the Python Software Foundation. -# No warranty expressed or implied. See the file LICENSE.PSF-2 for details. - -""" Distribution setup for python-daemon library. - """ - -import textwrap -from setuptools import setup, find_packages - -distribution_name = "python-daemon" -main_module_name = 'daemon' -main_module = __import__(main_module_name, fromlist=['version']) -version = main_module.version - -short_description, long_description = ( - textwrap.dedent(d).strip() - for d in main_module.__doc__.split(u'\n\n', 1) - ) - - -setup( - name=distribution_name, - version=version.version, - packages=find_packages(exclude=["test"]), - - # setuptools metadata - zip_safe=False, - test_suite="test.suite", - tests_require=[ - "MiniMock >=1.2.2", - ], - install_requires=[ - #"setuptools", - #"lockfile >=0.7", - ], - - # PyPI metadata - author=version.author_name, - author_email=version.author_email, - description=short_description, - license=version.license, - keywords=u"daemon fork unix".split(), - url=main_module._url, - long_description=long_description, - classifiers=[ - # Reference: http://pypi.python.org/pypi?%3Aaction=list_classifiers - "Development Status :: 4 - Beta", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX", - "Programming Language :: Python", - "Intended Audience :: Developers", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - ) diff --git a/scripts/scramble/scripts/DRMAA_python-macosx.py b/scripts/scramble/scripts/DRMAA_python-macosx.py deleted file mode 100644 index 18cc332aeaa..00000000000 --- a/scripts/scramble/scripts/DRMAA_python-macosx.py +++ /dev/null @@ -1,49 +0,0 @@ -import os, sys, shutil - -if "SGE_ROOT" not in os.environ: - print "main(): Please set SGE_ROOT to the path of your SGE installation" - print "main(): before scrambling DRMAA_python" - sys.exit(1) - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# patch -file = "setup.py" -print "main(): Patching", file -if not os.access( "%s.orig" %file, os.F_OK ): - shutil.copyfile( file, "%s.orig" %file ) -i = open( "%s.orig" %file, "r" ) -o = open( file, "w" ) -for line in i.readlines(): - if line == 'SGE6_ROOT="/scratch_test02/SGE6"\n': - line = 'SGE6_ROOT="%s"\n' % os.environ["SGE_ROOT"] - if line.startswith('link_args ='): - line = 'link_args = [ "-L%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-ldrmaa" ]\n' - print >>o, line, -i.close() -o.close() - -# build -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "build" ) -execfile( "setup.py", globals(), locals() ) - -# fix _cDRMAA.so rpath -so = "build/lib.%s-%s/_cDRMAA.so" % ( pkg_resources.get_platform(), sys.version[:3] ) -libdrmaa = os.path.join(SGE6_ROOT, "lib", SGE6_ARCH, "libdrmaa.dylib.1.0" ) -os.system( "install_name_tool -change libdrmaa.dylib.1.0 %s %s" % ( libdrmaa, so ) ) - -# package -sys.argv = [ me ] -sys.argv.append( "bdist_egg" ) -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/DRMAA_python-solaris.py b/scripts/scramble/scripts/DRMAA_python-solaris.py deleted file mode 100644 index 2eb3fbdad01..00000000000 --- a/scripts/scramble/scripts/DRMAA_python-solaris.py +++ /dev/null @@ -1,61 +0,0 @@ -import os, sys, shutil - -if "SGE_ROOT" not in os.environ: - print "main(): Please set SGE_ROOT to the path of your SGE installation" - print "main(): before scrambling DRMAA_python" - sys.exit(1) - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# if our python is 64 bit, use 64 bit sge... -if sys.maxint < ( 2048 * 1024 * 1024 ): - if sys.byteorder == 'big': - arch = "sparc" - else: - arch = "x86" -else: - if sys.byteorder == 'big': - arch = "sparc64" - else: - arch = "amd64" - -# if we're using sun cc, drop the gcc -Wno-unused option -cc = get_solaris_compiler() - -# patch -file = "setup.py" -print "main(): Patching", file -if not os.access( "%s.orig" %file, os.F_OK ): - shutil.copyfile( file, "%s.orig" %file ) -i = open( "%s.orig" %file, "r" ) -o = open( file, "w" ) -for line in i.readlines(): - if line == 'SGE6_ROOT="/scratch_test02/SGE6"\n': - line = 'SGE6_ROOT="%s"\n' % os.environ["SGE_ROOT"] - elif line.startswith('SGE6_ARCH='): - line = 'SGE6_ARCH="sol-%s"\n' % arch - elif line.startswith('link_args ='): - line = 'link_args = [ "-L%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-Wl,-R%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-ldrmaa" ]\n' - if line == " + [ '-Wno-unused' ]\n": - if cc == 'cc': - line = " #+ [ '-Wno-unused' ]\n" - print >>o, line, -i.close() -o.close() - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "bdist_egg" ) - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/DRMAA_python.py b/scripts/scramble/scripts/DRMAA_python.py deleted file mode 100644 index 4f2062e1451..00000000000 --- a/scripts/scramble/scripts/DRMAA_python.py +++ /dev/null @@ -1,41 +0,0 @@ -import os, sys, shutil - -if "SGE_ROOT" not in os.environ: - print "main(): Please set SGE_ROOT to the path of your SGE installation" - print "main(): before scrambling DRMAA_python" - sys.exit(1) - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# patch -file = "setup.py" -print "main(): Patching", file -if not os.access( "%s.orig" %file, os.F_OK ): - shutil.copyfile( file, "%s.orig" %file ) -i = open( "%s.orig" %file, "r" ) -o = open( file, "w" ) -for line in i.readlines(): - if line == 'SGE6_ROOT="/scratch_test02/SGE6"\n': - line = 'SGE6_ROOT="%s"\n' % os.environ["SGE_ROOT"] - if line.startswith('link_args ='): - line = 'link_args = [ "-L%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-Wl,-R%s" % os.path.join(SGE6_ROOT, "lib", SGE6_ARCH), "-ldrmaa" ]\n' - print >>o, line, -i.close() -o.close() - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "bdist_egg" ) - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/MySQL_python-solaris.py b/scripts/scramble/scripts/MySQL_python-solaris.py deleted file mode 100644 index 8ef903e90ce..00000000000 --- a/scripts/scramble/scripts/MySQL_python-solaris.py +++ /dev/null @@ -1,81 +0,0 @@ -import os, sys, shutil -from distutils.sysconfig import get_config_var - -def prep_mysql( prepped, args ): - - my_version = args['version'] - my_srcdir = os.path.join( os.getcwd(), "mysql-%s" % my_version ) - - # set up environment - os.environ['CC'] = get_config_var('CC') - os.environ['CFLAGS'] = get_config_var('CFLAGS') - os.environ['LDFLAGS'] = get_config_var('LDFLAGS') - - cc = get_solaris_compiler() - if cc == 'cc': - os.environ['CFLAGS'] += ' -KPIC' - elif cc == 'gcc': - os.environ['CFLAGS'] += ' -fPIC -DPIC' - - # run configure - run( "./configure --prefix=%s/mysql --disable-dependency-tracking --enable-static --disable-shared --without-server --without-uca " %os.getcwd() + \ - "--without-libwrap --without-ssl --without-docs --without-man --enable-thread-safe-client --with-named-curses-libs=''", - my_srcdir, "Configuring mysql (./configure)" ) - - # compile - run( "make", os.path.join( my_srcdir, 'include' ), "Preparing mysql includes (cd include; make)" ) - run( "make link_sources", os.path.join( my_srcdir, 'libmysql' ), "Preparing libmysql (cd libmysql; make link_sources)" ) - run( "make link_sources", os.path.join( my_srcdir, 'libmysql_r' ), "Preparing libmysql_r (cd libmysql_r; make link_sources)" ) - run( "make", os.path.join( my_srcdir, 'libmysql_r' ), "Compiling libmysql_r (cd libmysql_r; make)" ) - run( "make", os.path.join( my_srcdir, 'scripts' ), "Preparing scripts (cd scripts; make)" ) - - # install - run( "make install", os.path.join( my_srcdir, 'include' ), "Installing mysql includes (cd include; make install)" ) - run( "make install", os.path.join( my_srcdir, 'libmysql_r' ), "Installing libmysql_r (cd libmysql_r; make install)" ) - run( "make install", os.path.join( my_srcdir, 'scripts' ), "Installing mysql scripts (cd scripts; make install)" ) - shutil.copy( os.path.join( my_srcdir, 'include', 'mysqld_error.h' ), os.path.join( 'mysql', 'include', 'mysql' ) ) - - # create prepped archive - print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name - print " ", prepped - compress( prepped, - 'mysql/bin/mysql_config', - 'mysql/include', - 'mysql/lib' ) - -if __name__ == '__main__': - - # change back to the build dir - if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - - # find setuptools - sys.path.insert( 1, os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) - from scramble_lib import * - - tag = get_tag() - - my_version = ( tag.split( '_' ) )[1] - my_archive_base = os.path.join( archives, 'mysql-%s' % my_version ) - my_archive = get_archive( my_archive_base ) - my_archive_prepped = os.path.join( archives, 'mysql-%s-%s.tar.gz' % ( my_version, platform_noucs ) ) - - # clean up any existing stuff (could happen if you run scramble.py by hand) - clean( [ 'mysql-%s' % my_version, 'mysql' ] ) - - # unpack mysql - unpack_dep( my_archive, my_archive_prepped, prep_mysql, dict( version=my_version ) ) - - # get site.cfg - shutil.copy( os.path.join( patches, 'MySQL_python', 'site.cfg' ), 'site.cfg' ) - - # tag - me = sys.argv[0] - sys.argv = [ me ] - if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) - sys.argv.append( "bdist_egg" ) - - # go - execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/MySQL_python.py b/scripts/scramble/scripts/MySQL_python.py deleted file mode 100644 index 496689b3ba8..00000000000 --- a/scripts/scramble/scripts/MySQL_python.py +++ /dev/null @@ -1,78 +0,0 @@ -import os, sys, shutil -from distutils.sysconfig import get_config_var - -def prep_mysql( prepped, args ): - - my_version = args['version'] - my_srcdir = os.path.join( os.getcwd(), "mysql-%s" % my_version ) - - # set up environment - os.environ['CC'] = get_config_var('CC') - os.environ['CFLAGS'] = get_config_var('CFLAGS') - os.environ['LDFLAGS'] = get_config_var('LDFLAGS') - - if '-fPIC' not in os.environ['CFLAGS']: - os.environ['CFLAGS'] += ' -fPIC' - - # run configure - run( "./configure --prefix=%s/mysql --disable-dependency-tracking --enable-static --disable-shared --without-server --without-uca " %os.getcwd() + \ - "--without-libwrap --without-ssl --without-docs --without-man --enable-thread-safe-client --with-named-curses-libs=''", - my_srcdir, "Configuring mysql (./configure)" ) - - # compile - run( "make", os.path.join( my_srcdir, 'include' ), "Preparing mysql includes (cd include; make)" ) - run( "make link_sources", os.path.join( my_srcdir, 'libmysql' ), "Preparing libmysql (cd libmysql; make link_sources)" ) - run( "make link_sources", os.path.join( my_srcdir, 'libmysql_r' ), "Preparing libmysql_r (cd libmysql_r; make link_sources)" ) - run( "make", os.path.join( my_srcdir, 'libmysql_r' ), "Compiling libmysql_r (cd libmysql_r; make)" ) - run( "make", os.path.join( my_srcdir, 'scripts' ), "Preparing scripts (cd scripts; make)" ) - - # install - run( "make install", os.path.join( my_srcdir, 'include' ), "Installing mysql includes (cd include; make install)" ) - run( "make install", os.path.join( my_srcdir, 'libmysql_r' ), "Installing libmysql_r (cd libmysql_r; make install)" ) - run( "make install", os.path.join( my_srcdir, 'scripts' ), "Installing mysql scripts (cd scripts; make install)" ) - shutil.copy( os.path.join( my_srcdir, 'include', 'mysqld_error.h' ), os.path.join( 'mysql', 'include', 'mysql' ) ) - - # create prepped archive - print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name - print " ", prepped - compress( prepped, - 'mysql/bin/mysql_config', - 'mysql/include', - 'mysql/lib' ) - -if __name__ == '__main__': - - # change back to the build dir - if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - - # find setuptools - sys.path.insert( 1, os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) - from scramble_lib import * - - tag = get_tag() - - my_version = ( tag.split( '_' ) )[1] - my_archive_base = os.path.join( archives, 'mysql-%s' % my_version ) - my_archive = get_archive( my_archive_base ) - my_archive_prepped = os.path.join( archives, 'mysql-%s-%s.tar.gz' % ( my_version, platform_noucs ) ) - - # clean up any existing stuff (could happen if you run scramble.py by hand) - clean( [ 'mysql-%s' % my_version, 'mysql' ] ) - - # unpack mysql - unpack_dep( my_archive, my_archive_prepped, prep_mysql, dict( version=my_version ) ) - - # get site.cfg - shutil.copy( os.path.join( patches, 'MySQL_python', 'site.cfg' ), 'site.cfg' ) - - # tag - me = sys.argv[0] - sys.argv = [ me ] - if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) - sys.argv.append( "bdist_egg" ) - - # go - execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/PyRods.py b/scripts/scramble/scripts/PyRods.py deleted file mode 100644 index 3314ad9542c..00000000000 --- a/scripts/scramble/scripts/PyRods.py +++ /dev/null @@ -1,33 +0,0 @@ -import os, sys -import subprocess - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -get_deps() # require any dependent eggs -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -subprocess.check_call( "./scripts/configure", shell=True ) -subprocess.check_call( "CFLAGS='-fPIC' make clients", shell=True ) - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -# apply patches (if any) -apply_patches() - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/bx_python-solaris.py b/scripts/scramble/scripts/bx_python-solaris.py deleted file mode 100644 index 35adfe4b32a..00000000000 --- a/scripts/scramble/scripts/bx_python-solaris.py +++ /dev/null @@ -1,33 +0,0 @@ -import os, sys - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -get_deps() # require any dependent eggs -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -# apply patches (if any) -apply_patches() - -cc = get_solaris_compiler() -if cc == 'cc': - os.environ['CFLAGS'] = '-xc99=all -D_X_OPEN_SOURCE=600 -D_XPG6=1' - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/generic.py b/scripts/scramble/scripts/generic.py deleted file mode 100644 index ba929d636ad..00000000000 --- a/scripts/scramble/scripts/generic.py +++ /dev/null @@ -1,29 +0,0 @@ -import os, sys - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -get_deps() # require any dependent eggs -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -# apply patches (if any) -apply_patches() - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/pbs_python.py b/scripts/scramble/scripts/pbs_python.py deleted file mode 100644 index 2264530a975..00000000000 --- a/scripts/scramble/scripts/pbs_python.py +++ /dev/null @@ -1,49 +0,0 @@ -import os, sys, shutil - -if "LIBTORQUE_DIR" not in os.environ: - print "main(): Please set LIBTORQUE_DIR to the path of the" - print "main(): directory containing libtorque.so" - sys.exit(1) - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# the build process doesn't set an rpath for libtorque -os.environ['LD_RUN_PATH'] = os.environ['LIBTORQUE_DIR'] - -# run the config script -run( 'sh configure --with-pbsdir=%s' % os.environ['LIBTORQUE_DIR'], os.getcwd(), 'Running pbs_python configure script' ) - -# version string in 2.9.4 setup.py is wrong -print "scramble(): Patching setup.py" -if not os.path.exists( 'setup.py.orig' ): - shutil.copyfile( 'setup.py', 'setup.py.orig' ) - i = open( 'setup.py.orig', 'r' ) - o = open( 'setup.py', 'w' ) - for line in i.readlines(): - if line == " version = '4.0.0',\n": - line = " version = '4.1.0',\n" - print >>o, line, -i.close() -o.close() - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -# go -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/psycopg2-linux.py b/scripts/scramble/scripts/psycopg2-linux.py deleted file mode 100644 index 7c91b1da99d..00000000000 --- a/scripts/scramble/scripts/psycopg2-linux.py +++ /dev/null @@ -1,80 +0,0 @@ -import os, sys, shutil -from distutils.sysconfig import get_config_var - -def prep_postgres( prepped, args ): - - pg_version = args['version'] - pg_srcdir = os.path.join( os.getcwd(), "postgresql-%s" % pg_version ) - - # set up environment - os.environ['CC'] = get_config_var('CC') - os.environ['CFLAGS'] = get_config_var('CFLAGS') - os.environ['LDFLAGS'] = get_config_var('LDFLAGS') - - if '-fPIC' not in os.environ['CFLAGS']: - os.environ['CFLAGS'] += ' -fPIC' - - # run configure - run( "./configure --prefix=%s/postgres --disable-dependency-tracking --enable-static --disable-shared --without-readline --with-thread-safety" % os.getcwd(), - os.path.join( os.getcwd(), "postgresql-%s" % pg_version ), - "Configuring postgres (./configure)" ) - - # compile - run( "make ../../src/include/utils/fmgroids.h", os.path.join( pg_srcdir, 'src', 'backend' ), "Compiling fmgroids.h (cd src/backend; make ../../src/include/utils/fmgroids.h)" ) - run( "make", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; make)" ) - run( "make", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; make)" ) - - # install - run( "make install", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; make install)" ) - run( "make install", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; make install)" ) - run( "make install", os.path.join( pg_srcdir, 'src', 'include' ), "Compiling pg_config (cd src/include; make install)" ) - - # create prepped archive - print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name - print " ", prepped - compress( prepped, - 'postgres/bin', - 'postgres/include', - 'postgres/lib' ) - -if __name__ == '__main__': - - # change back to the build dir - if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - - # find setuptools - sys.path.insert( 1, os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) - from scramble_lib import * - - tag = get_tag() - - pg_version = ( tag.split( "_" ) )[1] - pg_archive_base = os.path.join( archives, "postgresql-%s" % pg_version ) - pg_archive = get_archive( pg_archive_base ) - pg_archive_prepped = os.path.join( archives, "postgresql-%s-%s.tar.gz" % ( pg_version, platform_noucs ) ) - - # clean up any existing stuff (could happen if you run scramble.py by hand) - clean( [ 'postgresql-%s' % pg_version ] ) - - # unpack postgres - unpack_dep( pg_archive, pg_archive_prepped, prep_postgres, dict( version=pg_version ) ) - - # localize setup.cfg - if not os.path.exists( 'setup.cfg.orig' ): - shutil.copy( 'setup.cfg', 'setup.cfg.orig' ) - f = open( 'setup.cfg', 'a' ) - f.write( '\npg_config=postgres/bin/pg_config\n' ) - f.write( '\nlibraries=crypt\n' ) - f.close() - - # tag - me = sys.argv[0] - sys.argv = [ me ] - if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) - sys.argv.append( "bdist_egg" ) - - # go - execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/psycopg2-macosx.py b/scripts/scramble/scripts/psycopg2-macosx.py deleted file mode 100644 index 77dc699ddcb..00000000000 --- a/scripts/scramble/scripts/psycopg2-macosx.py +++ /dev/null @@ -1,85 +0,0 @@ -import os, sys, shutil -from distutils.sysconfig import get_config_var - -def prep_postgres( prepped, args ): - - pg_version = args['version'] - pg_srcdir = os.path.join( os.getcwd(), "postgresql-%s" % pg_version ) - - # set up environment - os.environ['CC'] = get_config_var('CC') - os.environ['CFLAGS'] = get_config_var('CFLAGS') - os.environ['LDFLAGS'] = get_config_var('LDFLAGS') - - if '-fPIC' not in os.environ['CFLAGS']: - os.environ['CFLAGS'] += ' -fPIC' - - # run configure - run( "./configure --prefix=%s/postgres --disable-dependency-tracking --enable-static --disable-shared --without-readline --with-thread-safety" % os.getcwd(), - os.path.join( os.getcwd(), "postgresql-%s" % pg_version ), - "Configuring postgres (./configure)" ) - - # compile - run( "make ../../src/include/utils/fmgroids.h", os.path.join( pg_srcdir, 'src', 'backend' ), "Compiling fmgroids.h (cd src/backend; make ../../src/include/utils/fmgroids.h)" ) - run( "make all-static-lib", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; make)" ) - run( "make", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; make)" ) - - # install - run( "make install-lib-static", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; make install)" ) - run( "make install", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; make install)" ) - run( "make install", os.path.join( pg_srcdir, 'src', 'include' ), "Compiling pg_config (cd src/include; make install)" ) - - # manually install some headers - run( "cp libpq-fe.h %s" % os.path.join( os.getcwd(), 'postgres', 'include' ), os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Installing libpq-fe.h" ) - run( "cp libpq-events.h %s" % os.path.join( os.getcwd(), 'postgres', 'include' ), os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Installing libpq-fe.h" ) - run( "cp libpq-int.h %s" % os.path.join( os.getcwd(), 'postgres', 'include', 'internal' ), os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Installing libpq-fe.h" ) - run( "cp pqexpbuffer.h %s" % os.path.join( os.getcwd(), 'postgres', 'include', 'internal' ), os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Installing libpq-fe.h" ) - - # create prepped archive - print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name - print " ", prepped - compress( prepped, - 'postgres/bin', - 'postgres/include', - 'postgres/lib' ) - -if __name__ == '__main__': - - # change back to the build dir - if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - - # find setuptools - sys.path.insert( 1, os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) - from scramble_lib import * - - tag = get_tag() - - pg_version = ( tag.split( "_" ) )[1] - pg_archive_base = os.path.join( archives, "postgresql-%s" % pg_version ) - pg_archive = get_archive( pg_archive_base ) - pg_archive_prepped = os.path.join( archives, "postgresql-%s-%s.tar.gz" % ( pg_version, platform_noucs ) ) - - # clean up any existing stuff (could happen if you run scramble.py by hand) - clean( [ 'postgresql-%s' % pg_version ] ) - - # unpack postgres - unpack_dep( pg_archive, pg_archive_prepped, prep_postgres, dict( version=pg_version ) ) - - # localize setup.cfg - if not os.path.exists( 'setup.cfg.orig' ): - shutil.copy( 'setup.cfg', 'setup.cfg.orig' ) - f = open( 'setup.cfg', 'a' ) - f.write( '\npg_config=postgres/bin/pg_config\n' ) - f.close() - - # tag - me = sys.argv[0] - sys.argv = [ me ] - if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) - sys.argv.append( "bdist_egg" ) - - # go - execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/psycopg2-solaris.py b/scripts/scramble/scripts/psycopg2-solaris.py deleted file mode 100644 index ff0050558eb..00000000000 --- a/scripts/scramble/scripts/psycopg2-solaris.py +++ /dev/null @@ -1,82 +0,0 @@ -import os, sys, shutil -from distutils.sysconfig import get_config_var - -def prep_postgres( prepped, args ): - - pg_version = args['version'] - pg_srcdir = os.path.join( os.getcwd(), "postgresql-%s" % pg_version ) - - # set up environment - os.environ['CC'] = get_config_var('CC') - os.environ['CFLAGS'] = get_config_var('CFLAGS') - os.environ['LDFLAGS'] = get_config_var('LDFLAGS') - - cc = get_solaris_compiler() - if cc == 'cc': - os.environ['CFLAGS'] += ' -KPIC' - elif cc == 'gcc': - os.environ['CFLAGS'] += ' -fPIC -DPIC' - - # run configure - run( "./configure --prefix=%s/postgres --disable-dependency-tracking --enable-static --disable-shared --without-readline --with-thread-safety" % os.getcwd(), - os.path.join( os.getcwd(), "postgresql-%s" % pg_version ), - "Configuring postgres (./configure)" ) - - # compile - run( "gmake ../../src/include/utils/fmgroids.h", os.path.join( pg_srcdir, 'src', 'backend' ), "Compiling fmgroids.h (cd src/backend; gmake ../../src/include/utils/fmgroids.h)" ) - run( "gmake", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; gmake)" ) - run( "gmake", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; gmake)" ) - - # install - run( "gmake install", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; gmake install)" ) - run( "gmake install", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; gmake install)" ) - run( "gmake install", os.path.join( pg_srcdir, 'src', 'include' ), "Compiling pg_config (cd src/include; gmake install)" ) - - # create prepped archive - print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name - print " ", prepped - compress( prepped, - 'postgres/bin', - 'postgres/include', - 'postgres/lib' ) - -if __name__ == '__main__': - - # change back to the build dir - if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - - # find setuptools - sys.path.insert( 1, os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) - from scramble_lib import * - - tag = get_tag() - - pg_version = ( tag.split( "_" ) )[1] - pg_archive_base = os.path.join( archives, "postgresql-%s" % pg_version ) - pg_archive = get_archive( pg_archive_base ) - pg_archive_prepped = os.path.join( archives, "postgresql-%s-%s.tar.gz" % ( pg_version, platform_noucs ) ) - - # clean up any existing stuff (could happen if you run scramble.py by hand) - clean( [ 'postgresql-%s' % pg_version ] ) - - # unpack postgres - unpack_dep( pg_archive, pg_archive_prepped, prep_postgres, dict( version=pg_version ) ) - - # localize setup.cfg - if not os.path.exists( 'setup.cfg.orig' ): - shutil.copy( 'setup.cfg', 'setup.cfg.orig' ) - f = open( 'setup.cfg', 'a' ) - f.write( '\npg_config=postgres/bin/pg_config\n' ) - f.close() - - # tag - me = sys.argv[0] - sys.argv = [ me ] - if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) - sys.argv.append( "bdist_egg" ) - - # go - execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/psycopg2.py b/scripts/scramble/scripts/psycopg2.py deleted file mode 100644 index bcaa2081f2c..00000000000 --- a/scripts/scramble/scripts/psycopg2.py +++ /dev/null @@ -1,79 +0,0 @@ -import os, sys, shutil -from distutils.sysconfig import get_config_var - -def prep_postgres( prepped, args ): - - pg_version = args['version'] - pg_srcdir = os.path.join( os.getcwd(), "postgresql-%s" % pg_version ) - - # set up environment - os.environ['CC'] = get_config_var('CC') - os.environ['CFLAGS'] = get_config_var('CFLAGS') - os.environ['LDFLAGS'] = get_config_var('LDFLAGS') - - if '-fPIC' not in os.environ['CFLAGS']: - os.environ['CFLAGS'] += ' -fPIC' - - # run configure - run( "./configure --prefix=%s/postgres --disable-dependency-tracking --enable-static --disable-shared --without-readline --with-thread-safety" % os.getcwd(), - os.path.join( os.getcwd(), "postgresql-%s" % pg_version ), - "Configuring postgres (./configure)" ) - - # compile - run( "make ../../src/include/utils/fmgroids.h", os.path.join( pg_srcdir, 'src', 'backend' ), "Compiling fmgroids.h (cd src/backend; make ../../src/include/utils/fmgroids.h)" ) - run( "make", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; make)" ) - run( "make", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; make)" ) - - # install - run( "make install", os.path.join( pg_srcdir, 'src', 'interfaces', 'libpq' ), "Compiling libpq (cd src/interfaces/libpq; make install)" ) - run( "make install", os.path.join( pg_srcdir, 'src', 'bin', 'pg_config' ), "Compiling pg_config (cd src/bin/pg_config; make install)" ) - run( "make install", os.path.join( pg_srcdir, 'src', 'include' ), "Compiling pg_config (cd src/include; make install)" ) - - # create prepped archive - print "%s(): Creating prepped archive for future builds at:" % sys._getframe().f_code.co_name - print " ", prepped - compress( prepped, - 'postgres/bin', - 'postgres/include', - 'postgres/lib' ) - -if __name__ == '__main__': - - # change back to the build dir - if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - - # find setuptools - sys.path.insert( 1, os.path.abspath( os.path.join( '..', '..', '..', 'lib' ) ) ) - from scramble_lib import * - - tag = get_tag() - - pg_version = ( tag.split( "_" ) )[1] - pg_archive_base = os.path.join( archives, "postgresql-%s" % pg_version ) - pg_archive = get_archive( pg_archive_base ) - pg_archive_prepped = os.path.join( archives, "postgresql-%s-%s.tar.gz" % ( pg_version, platform_noucs ) ) - - # clean up any existing stuff (could happen if you run scramble.py by hand) - clean( [ 'postgresql-%s' % pg_version ] ) - - # unpack postgres - unpack_dep( pg_archive, pg_archive_prepped, prep_postgres, dict( version=pg_version ) ) - - # localize setup.cfg - if not os.path.exists( 'setup.cfg.orig' ): - shutil.copy( 'setup.cfg', 'setup.cfg.orig' ) - f = open( 'setup.cfg', 'a' ) - f.write( '\npg_config=postgres/bin/pg_config\n' ) - f.close() - - # tag - me = sys.argv[0] - sys.argv = [ me ] - if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) - sys.argv.append( "bdist_egg" ) - - # go - execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/pysam-solaris.py b/scripts/scramble/scripts/pysam-solaris.py deleted file mode 100644 index 6b4d62c5470..00000000000 --- a/scripts/scramble/scripts/pysam-solaris.py +++ /dev/null @@ -1,30 +0,0 @@ -import os, sys - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag -clean() # clean up any existing stuff (could happen if you run scramble.py by hand) - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "egg_info" ) -if tag is not None: - sys.argv.append( "--tag-build=%s" %tag ) -# svn revision (if any) is handled directly in tag-build -sys.argv.append( "--no-svn-revision" ) -sys.argv.append( "bdist_egg" ) - -cc = get_solaris_compiler() -if cc == 'cc': - os.environ['CFLAGS'] = '-xc99=all' - os.environ['LDFLAGS'] = '-lsocket -lnsl' - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/pysqlite.py b/scripts/scramble/scripts/pysqlite.py deleted file mode 100644 index 4a6e277327d..00000000000 --- a/scripts/scramble/scripts/pysqlite.py +++ /dev/null @@ -1,36 +0,0 @@ -import os, sys - -def prep_sqlite( prepped, args ): - os.makedirs( 'amalgamation' ) - for file in ( 'sqlite3.h', 'sqlite3.c', 'sqlite3ext.h' ): - os.rename( file, os.path.join( 'amalgamation', file ) ) - -# change back to the build dir -if os.path.dirname( sys.argv[0] ) != "": - os.chdir( os.path.dirname( sys.argv[0] ) ) - -# find setuptools -sys.path.insert( 1, os.path.join( '..', '..', '..', 'lib' ) ) -from scramble_lib import * - -tag = get_tag() # get the tag - -sqlite_version = ( tag.split( "_" ) )[1].replace('.','_') -sqlite_archive_base = os.path.join( archives, "sqlite-amalgamation-%s" % sqlite_version ) -sqlite_archive = get_archive( sqlite_archive_base ) - -clean( ['amalgamation'] ) # clean up any existing stuff (could happen if you run scramble.py by hand) - -unpack_dep( sqlite_archive, None, prep_sqlite, None ) - -# reset args for distutils -me = sys.argv[0] -sys.argv = [ me ] -sys.argv.append( "build_static" ) -if tag is not None: - sys.argv.append( "egg_info" ) - sys.argv.append( "--tag-build=%s" %tag ) -sys.argv.append( "bdist_egg" ) - -# do it -execfile( "setup.py", globals(), locals() ) diff --git a/scripts/test_dist_egg.py b/scripts/test_dist_egg.py deleted file mode 100644 index 5ce6475f27a..00000000000 --- a/scripts/test_dist_egg.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -""" -A crude script for minimal "testing" of dist eggs (require and import). It may -not work on all zipped eggs. It may be easiest to just customize this script -for whatever egg you want to test. - -usage: test_dist_egg.py -""" -import os, sys, logging, subprocess - -try: - assert sys.argv[1] -except: - print __doc__ - sys.exit( 1 ) - -lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) -sys.path.insert( 0, lib ) - -if sys.argv[1].endswith( '.egg' ): - - egg = sys.argv[1] - egg_name = os.path.basename( egg ).split( '-' )[0] - sys.path.insert( 0, egg ) - - import pkg_resources - pkg_resources.require( egg_name ) - provider = pkg_resources.get_provider( egg_name ) - importables = provider.get_metadata('top_level.txt').splitlines() - - for importable in importables: - mod = __import__( importable ) - assert os.path.dirname( mod.__path__[0] ) == os.path.dirname( provider.module_path ) - print "OK" - - sys.exit( 0 ) - -else: - - build_dir = os.path.join( os.path.dirname( os.path.abspath( __file__ ) ), 'scramble', 'build' ) - if os.path.exists( build_dir ): - raise Exception( 'Build dir must be removed before testing: %s' % build_dir ) - - name = sys.argv[1] - - from galaxy.eggs.dist import DistScrambleCrate - - c = DistScrambleCrate() - - for egg in c[name]: - print 'Checking %s %s for %s on %s' % ( name, egg.version, egg.platform, egg.build_host ) - p = subprocess.Popen( 'ssh %s %s %s %s %s' % ( egg.build_host, egg.python, os.path.abspath( __file__ ), egg.distribution.location, egg.platform ), shell=True ) - p.wait() From a24134c447818fc8f55e0f7b7a0f33690155444b Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 5 Jul 2015 05:24:15 -0400 Subject: [PATCH 02/52] Install modified pip for fetching Linux wheels, fetch wheels from wheels.galaxyproject.org --- scripts/common_startup.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index e6bfbcdbeab..ced1d43565f 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -87,8 +87,13 @@ if [ $SET_VENV -eq 1 ]; then fi fi +if [ -z "$VIRTUAL_ENV" ]; then + echo "ERROR: A virtualenv cannot be found or created. Please create a virtualenv in .venv, or activate one." + exit 1 +fi + if [ $FETCH_WHEELS -eq 1 ]; then - PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.requirements('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin - #pip install -r requirements.txt --index-url https://wheels.galaxyproject.org/ + pip install git+https://github.com/natefoo/pip@linux-wheels + PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.requirements('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin --index-url https://wheels.galaxyproject.org/simple/ fi From 0de745188e28b5b53097b7cb1d69454bf681f523 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 5 Jul 2015 05:47:09 -0400 Subject: [PATCH 03/52] Remove (dist-)eggs.ini. --- dist-eggs.ini | 52 ---------------------------- eggs.ini | 96 --------------------------------------------------- 2 files changed, 148 deletions(-) delete mode 100644 dist-eggs.ini delete mode 100644 eggs.ini diff --git a/dist-eggs.ini b/dist-eggs.ini deleted file mode 100644 index 92146d83bce..00000000000 --- a/dist-eggs.ini +++ /dev/null @@ -1,52 +0,0 @@ -; -; Config for building eggs for distribution (via a site such as -; eggs.galaxyproject.org). Probably only useful to members of the Galaxy Team -; building eggs for distribution. This file is used by -; scripts/dist-scramble.py -; -; More information: http://wiki.galaxyproject.org/Admin/Config/Eggs -; - -[hosts] -py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-i686-ucs2/bin/python2.6 -py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-i686-ucs4/bin/python2.6 -py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs2/bin/python2.6 -py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs4/bin/python2.6 -py2.7-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-i686-ucs2/bin/python2.7 -py2.7-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-i686-ucs4/bin/python2.7 -py2.7-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs2/bin/python2.7 -py2.7-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs4/bin/python2.7 -py2.6-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.6 -py2.7-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.7 -; -; these hosts are used to build eggs with no C extensions -py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs4/bin/python2.6 -py2.7 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs4/bin/python2.7 - -[groups] -py2.6-linux-i686 = py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 -py2.6-linux-x86_64 = py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 -py2.7-linux-i686 = py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 -py2.7-linux-x86_64 = py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 -py2.6-linux = py2.6-linux-i686 py2.6-linux-x86_64 -py2.7-linux = py2.7-linux-i686 py2.7-linux-x86_64 -linux-i686 = py2.6-linux-i686 py2.7-linux-i686 -linux-x86_64 = py2.6-linux-x86_64 py2.7-linux-x86_64 -linux = linux-i686 linux-x86_64 -py2.6-macosx = py2.6-macosx-10.6-intel-ucs2 -py2.7-macosx = py2.7-macosx-10.6-intel-ucs2 -macosx = py2.6-macosx py2.7-macosx -py2.6-all = py2.6-linux py2.6-macosx -py2.7-all = py2.7-linux py2.7-macosx - -; the 'all' key is used internally by the build system to specify which hosts -; to build on when no hosts are specified on the dist-eggs.py command line. -all = linux macosx - -; the 'noplatform' key, likewise, is for which build hosts should be used when -; building pure python (noplatform) eggs. -noplatform = py2.6 py2.7 - -; don't build these eggs on these platforms: -[ignore] -;ctypes = diff --git a/eggs.ini b/eggs.ini deleted file mode 100644 index 1219ea0c722..00000000000 --- a/eggs.ini +++ /dev/null @@ -1,96 +0,0 @@ -; -; Galaxy egg version configuration -; -; This file is version controlled and should not be edited by hand! -; For more information, see: -; https://wiki.galaxyproject.org/Admin/Config/Eggs -; - -[general] -repository = http://eggs.galaxyproject.org -; these eggs must be scrambled for your local environment -no_auto = pbs_python - -[eggs:platform] -bx_python = 0.7.2 -Cheetah = 2.2.2 -MarkupSafe = 0.12 -mercurial = 3.2.4 -MySQL_python = 1.2.3c1 -PyRods = 3.2.4 -numpy = 1.6.0 -pbs_python = 4.3.5 -psycopg2 = 2.5.1 -pycrypto = 2.5 -pysam = 0.4.2 -pysqlite = 2.5.6 -python_lzo = 1.08_2.03_static -PyYAML = 3.10 -SQLAlchemy = 1.0.1 -; msgpack_python = 0.2.4 - -[eggs:noplatform] -amqp = 1.4.6 -anyjson = 0.3.3 -Beaker = 1.4 -bioblend = 0.5.2 -boto = 2.27.0 -decorator = 3.1.2 -docutils = 0.7 -drmaa = 0.7.6 -Fabric = 1.7.0 -importlib = 1.0.3 -kombu = 3.0.24 -lrucache = 0.2 -Mako = 0.4.1 -mock = 1.0.1 -nose = 1.3.6 -NoseHTML = 0.4.1 -NoseTestDiff = 0.1 -ordereddict = 1.1 -paramiko = 1.11.1 -Parsley = 1.1 -Paste = 1.7.5.1 -PasteDeploy = 1.5.0 -pexpect = 2.4 -poster = 0.8.1 -python_openid = 2.2.5 -python_daemon = 1.5.5 -pytz = 2013.9 -repoze.lru = 0.6 -requests = 2.2.1 -Routes = 2.1 -simplejson = 2.1.1 -six = 1.9.0 -sqlalchemy_migrate = 0.9.6 -sqlparse = 0.1.14 -ssh = 1.7.14 -SVGFig = 1.1.6 -Tempita = 0.5.1 -twill = 0.9 -WebError = 0.8a -WebHelpers = 1.3 -WebOb = 0.8.5 -wsgiref = 0.1.2 -Babel = 1.3 -wchartype = 0.1 -Whoosh = 2.4.1 -; fluent_logger = 0.3.3 -raven = 3.1.8 - -; extra version information -[tags] -psycopg2 = _9.2.4_static -pysqlite = _3.6.17_static -MySQL_python = _5.1.41_static -pysam = _kanwei_b10f6e722e9a - -; dependency source urls, necessary for scrambling. for an explanation, see -; the wiki page above -[source] -MySQL_python = mysql-5.1.41 -psycopg2 = postgresql-9.2.4 -pysqlite = sqlite-amalgamation-3_6_17 - -[dependencies] -bx_python = numpy From 33095318f777bf55fcf01cff4fe9d00910ff2868 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 5 Jul 2015 05:52:40 -0400 Subject: [PATCH 04/52] Fix problems in wheels.py found by flake8 --- lib/galaxy/wheels.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/wheels.py b/lib/galaxy/wheels.py index b1d3b2bc0be..c94159779c4 100644 --- a/lib/galaxy/wheels.py +++ b/lib/galaxy/wheels.py @@ -1,6 +1,8 @@ """ """ +import sys + from os import pardir from os.path import abspath, join, dirname @@ -37,7 +39,7 @@ class GalaxyConfig( object ): # SQLite is different since it can be specified in two config vars and defaults to True if name == "pysqlite": try: - return config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) + return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) except: return True else: @@ -49,7 +51,6 @@ class GalaxyConfig( object ): "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), - "pysam": lambda: check_pysam(), "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" }.get( name, lambda: True )() except: From 55019173466628c117a8a41eb068a605c1282f06 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 5 Jul 2015 10:50:54 -0400 Subject: [PATCH 05/52] Install modified pip in develop/edit mode so it won't be reinstalled every time (courtesy @jxtx). --- scripts/common_startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index ced1d43565f..f23c09aa2cb 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -94,6 +94,6 @@ fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install git+https://github.com/natefoo/pip@linux-wheels + pip install -e git+https://github.com/natefoo/pip@linux-wheels#egg=pip PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.requirements('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin --index-url https://wheels.galaxyproject.org/simple/ fi From e6446b05d35e7950675f4bcd95cf8c9e1c89ef1d Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 5 Jul 2015 10:56:58 -0400 Subject: [PATCH 06/52] Add nose to requirements.txt. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index ea880f64cfd..9ee8016957e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -61,3 +61,4 @@ bioblend Parsley WebError http://svgfig.googlecode.com/files/svgfig-1.1.6.tgz +nose From 033b1e88da99b86ffbd33321374aa59d1d72484c Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 13 Jul 2015 15:37:59 -0400 Subject: [PATCH 07/52] Remove all imports of `galaxy.eggs` and all calls to `{eggs,pkg_resources}.require`. Probably broke some stuff where we were trying to be smart about whether a particular egg was available, but Galaxy does at least start. --- lib/galaxy/config.py | 2 -- lib/galaxy/datatypes/binary.py | 8 +---- .../bedgraph_to_array_tree_converter.py | 4 --- lib/galaxy/datatypes/converters/bgzip.py | 2 -- .../gff_to_interval_index_converter.py | 2 -- .../converters/interval_to_bed_converter.py | 2 -- .../interval_to_bedstrict_converter.py | 2 -- .../converters/interval_to_coverage.py | 2 -- .../datatypes/converters/interval_to_fli.py | 2 -- .../interval_to_interval_index_converter.py | 2 -- .../converters/interval_to_tabix_converter.py | 2 -- .../converters/maf_to_fasta_converter.py | 2 -- .../converters/maf_to_interval_converter.py | 2 -- .../converters/pbed_ldreduced_converter.py | 2 -- .../pileup_to_interval_index_converter.py | 2 -- .../vcf_to_interval_index_converter.py | 2 -- .../datatypes/converters/vcf_to_vcf_bgzip.py | 2 -- .../wiggle_to_array_tree_converter.py | 3 -- .../converters/wiggle_to_simple_converter.py | 2 -- lib/galaxy/datatypes/coverage.py | 3 -- lib/galaxy/datatypes/data.py | 2 -- lib/galaxy/datatypes/dataproviders/dataset.py | 2 -- lib/galaxy/datatypes/metadata.py | 2 -- lib/galaxy/datatypes/sequence.py | 8 ++--- lib/galaxy/exceptions/__init__.py | 3 -- lib/galaxy/jobs/actions/post.py | 2 -- lib/galaxy/jobs/deferred/data_transfer.py | 1 - lib/galaxy/jobs/runners/__init__.py | 9 ----- lib/galaxy/jobs/runners/drmaa.py | 3 -- .../jobs/runners/lwr_client/config_util.py | 12 ++----- lib/galaxy/jobs/runners/pbs.py | 18 +--------- lib/galaxy/jobs/transfer_manager.py | 1 - lib/galaxy/model/__init__.py | 3 -- lib/galaxy/model/custom_types.py | 2 -- lib/galaxy/model/migrate/check.py | 22 ------------ lib/galaxy/model/orm/__init__.py | 34 ------------------- lib/galaxy/model/orm/engine_factory.py | 3 -- lib/galaxy/model/orm/scripts.py | 26 -------------- lib/galaxy/model/search.py | 2 -- .../model/tool_shed_install/migrate/check.py | 22 ------------ lib/galaxy/objectstore/rods.py | 14 +------- lib/galaxy/objectstore/s3_multipart_upload.py | 13 ++----- lib/galaxy/queue_worker.py | 8 +---- lib/galaxy/queues.py | 8 ----- lib/galaxy/tools/__init__.py | 7 ---- lib/galaxy/tools/parser/factory.py | 6 ---- lib/galaxy/tools/search/__init__.py | 2 -- lib/galaxy/tools/toolbox/base.py | 2 -- lib/galaxy/util/__init__.py | 4 --- lib/galaxy/util/log/fluent_log.py | 14 +------- lib/galaxy/util/plugin_config.py | 14 +------- .../visualization/data_providers/genome.py | 4 --- lib/galaxy/web/framework/__init__.py | 4 --- lib/galaxy/web/framework/decorators.py | 2 -- lib/galaxy/web/framework/helpers/__init__.py | 5 --- lib/galaxy/web/framework/middleware/sentry.py | 2 -- lib/galaxy/web/framework/openid_manager.py | 3 -- lib/galaxy/web/framework/webapp.py | 7 ---- lib/galaxy/webapps/galaxy/buildapp.py | 3 -- .../webapps/galaxy/controllers/cloudlaunch.py | 6 ---- .../webapps/galaxy/controllers/dataset.py | 3 +- .../galaxy/controllers/library_common.py | 23 +++++-------- .../galaxy/controllers/requests_admin.py | 2 -- .../webapps/reports/controllers/workflows.py | 2 -- lib/galaxy/webapps/tool_shed/config.py | 2 -- .../webapps/tool_shed/controllers/hg.py | 2 -- .../tool_shed/controllers/repository.py | 3 -- .../webapps/tool_shed/controllers/upload.py | 2 -- .../tool_shed/framework/middleware/hg.py | 2 -- .../webapps/tool_shed/model/__init__.py | 2 -- .../webapps/tool_shed/model/migrate/check.py | 24 ------------- .../webapps/tool_shed/search/repo_search.py | 2 -- .../webapps/tool_shed/search/tool_search.py | 2 -- lib/galaxy/workflow/render.py | 3 -- lib/galaxy_ext/metadata/set_metadata.py | 2 -- lib/pulsar/client/config_util.py | 15 ++------ lib/pulsar/client/transport/poster.py | 18 +--------- lib/pulsar/client/transport/requests.py | 18 +--------- .../galaxy_install/install_manager.py | 5 --- lib/tool_shed/galaxy_install/migrate/check.py | 20 ----------- .../recipe/install_environment.py | 6 ---- .../tool_dependencies/recipe/step_handler.py | 6 ---- lib/tool_shed/grids/admin_grids.py | 3 -- lib/tool_shed/grids/repository_grids.py | 3 -- lib/tool_shed/repository_types/metadata.py | 2 -- .../repository_suite_definition.py | 2 -- .../tool_dependency_definition.py | 2 -- .../bootstrap_tool_shed/bootstrap_util.py | 2 -- .../create_user_with_api_key.py | 4 --- ...check_repositories_for_functional_tests.py | 4 --- ...mpty_tool_dependency_installation_paths.py | 2 -- ...tool_dependency_definition_repositories.py | 4 --- ...ol_dependency_installation_dir_contents.py | 3 -- lib/tool_shed/util/basic_util.py | 3 -- lib/tool_shed/util/encoding_util.py | 1 - lib/tool_shed/util/hg_util.py | 2 -- 96 files changed, 25 insertions(+), 531 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 52c65ea9741..c80d6f11ffd 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -14,7 +14,6 @@ import string import sys import tempfile from datetime import timedelta -from galaxy import eggs from galaxy.exceptions import ConfigurationError from galaxy.util import listify from galaxy.util import string_as_bool @@ -720,7 +719,6 @@ def configure_logging( config ): root.addHandler( handler ) # If sentry is configured, also log to it if config.sentry_dsn: - eggs.require( "raven" ) from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 4fd0caf968b..11f557e731d 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -15,9 +15,6 @@ import re import warnings import zipfile -from galaxy import eggs -eggs.require( "bx-python" ) - from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE from galaxy.util import sqlite @@ -25,10 +22,7 @@ from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListPa from galaxy.datatypes import metadata import dataproviders -with warnings.catch_warnings(): - warnings.simplefilter( "ignore" ) - eggs.require( "pysam" ) - from pysam import csamtools +from pysam import csamtools log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py index f66b1d8da27..5180863f127 100644 --- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py @@ -4,10 +4,6 @@ from __future__ import division import sys -from galaxy import eggs - -eggs.require('numpy') # noqa -eggs.require('bx-python') # noqa from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict BLOCK_SIZE = 100 diff --git a/lib/galaxy/datatypes/converters/bgzip.py b/lib/galaxy/datatypes/converters/bgzip.py index a419257a2b8..b6aeaa4a860 100644 --- a/lib/galaxy/datatypes/converters/bgzip.py +++ b/lib/galaxy/datatypes/converters/bgzip.py @@ -6,8 +6,6 @@ Uses pysam to bgzip a file usage: %prog in_file out_file """ -from galaxy import eggs -import pkg_resources; pkg_resources.require( "pysam" ) import ctabix, subprocess, tempfile, sys, optparse def main(): diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py index 07b3631dc71..5af0396e226 100644 --- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py @@ -10,8 +10,6 @@ usage: from __future__ import division import sys, fileinput -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) from galaxy.datatypes.util.gff_util import * from bx.interval_index_file import Indexes diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py index cfa48cedd28..6e4c90a9407 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py @@ -2,8 +2,6 @@ #Dan Blankenberg import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import bx.intervals.io assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py index 2fb612c4d66..98461f2967a 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py @@ -2,8 +2,6 @@ #Dan Blankenberg import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import bx.intervals.io assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index 64868ff019a..459be4ce7a8 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -7,8 +7,6 @@ usage: %prog bed_file out_file -2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file """ import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) from bx.intervals import io from bx.cookbook import doc_optparse import psyco_full diff --git a/lib/galaxy/datatypes/converters/interval_to_fli.py b/lib/galaxy/datatypes/converters/interval_to_fli.py index 22b429d1945..1a238fe3591 100644 --- a/lib/galaxy/datatypes/converters/interval_to_fli.py +++ b/lib/galaxy/datatypes/converters/interval_to_fli.py @@ -15,8 +15,6 @@ and symbols are sorted in lexigraphical order. ''' import sys, optparse -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) from bx.tabular.io import Comment from galaxy.datatypes.util.gff_util import GFFReaderWrapper, read_unordered_gtf, convert_gff_coords_to_bed diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py index 8d1b7653cb0..43937421416 100644 --- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py @@ -12,8 +12,6 @@ usage: %prog in_file out_file from __future__ import division import optparse -from galaxy import eggs -eggs.require( "bx-python" ) from bx.interval_index_file import Indexes def main(): diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py index 13fb2e53c73..7919dbd6a22 100644 --- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py @@ -7,8 +7,6 @@ Supported presets: bed, gff, vcf usage: %prog in_file out_file """ -from galaxy import eggs -import pkg_resources; pkg_resources.require( "pysam" ) import ctabix, subprocess, tempfile, sys, os, optparse def main(): diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index df62e347242..70bef1e283f 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -2,8 +2,6 @@ #Dan Blankenberg import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import bx.align.maf from galaxy.tools.util import maf_utilities diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index 2ddb00cbe43..21884307547 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -2,8 +2,6 @@ #Dan Blankenberg import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import bx.align.maf from galaxy.tools.util import maf_utilities diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 94f2f4fb62f..ff63189190b 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -4,8 +4,6 @@ import os,sys,tempfile,subprocess,time -from galaxy import eggs - prog="pbed_ldreduced_converter.py" galhtmlprefix = """ diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py index 88e76b40f33..1fa97057598 100644 --- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py @@ -9,8 +9,6 @@ usage: %prog in_file out_file from __future__ import division import optparse -from galaxy import eggs -eggs.require( "bx-python" ) from bx.interval_index_file import Indexes def main(): diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py index a55b7d2b5e4..6ce8a6fc6f9 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py @@ -7,8 +7,6 @@ Convert from VCF file to interval index file. from __future__ import division import optparse -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import galaxy_utils.sequence.vcf from bx.interval_index_file import Indexes diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py index d439aa7fc51..423b2eb8a5f 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py +++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py @@ -8,8 +8,6 @@ Original ordering, which may be specifically needed by tools or external displa usage: %prog in_file out_file """ -from galaxy import eggs -import pkg_resources; pkg_resources.require( "pysam" ) import ctabix, optparse def main(): diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py index 6332c4ead8a..f5201f6c290 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py @@ -4,9 +4,6 @@ from __future__ import division import sys -from galaxy import eggs -eggs.require('numpy') # noqa -eggs.require('bx-python') # noqa from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict from bx.arrays.wiggle import WiggleReader diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py index c70d2c9f459..cc119a0fd78 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py @@ -7,8 +7,6 @@ Read a wiggle track and print out a series of lines containing and fixedStep wiggle lines. """ import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import bx.wiggle from galaxy.tools.exception_handling import * diff --git a/lib/galaxy/datatypes/coverage.py b/lib/galaxy/datatypes/coverage.py index e5f78ba9b59..f0ef9203888 100644 --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -6,7 +6,6 @@ Coverage datatypes import logging import math -from galaxy import eggs from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.tabular import Tabular @@ -27,8 +26,6 @@ class LastzCoverage( Tabular ): Assumes we have a numpy file. """ # Maybe if we import here people will still be able to use Galaxy when numpy kills it - eggs.require("numpy>=1.2.1") - #from numpy.lib import format import numpy range = end - start diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 59e1384e816..4c8d44e0936 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -17,8 +17,6 @@ from galaxy.util.sanitize_html import sanitize_html import dataproviders -from galaxy import eggs -eggs.require( "Paste" ) import paste XSS_VULNERABLE_MIME_TYPES = [ diff --git a/lib/galaxy/datatypes/dataproviders/dataset.py b/lib/galaxy/datatypes/dataproviders/dataset.py index c7a279eda8e..ba9278ca1f9 100644 --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -15,8 +15,6 @@ from galaxy.util import sqlite import re import sys -from galaxy import eggs -eggs.require( 'bx-python' ) from bx import seq as bx_seq from bx import wiggle as bx_wig from bx import bbi as bx_bbi diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 85e88bcace9..120801d10e0 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -14,8 +14,6 @@ import weakref from os.path import abspath -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy.orm import object_session import galaxy.model diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 46c238cb430..6616b200c73 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -12,18 +12,14 @@ import string from cgi import escape -from galaxy import eggs, util +from galaxy import util from galaxy.datatypes import metadata from galaxy.datatypes.checkers import is_gzip from galaxy.datatypes.sniff import get_test_fname, get_headers from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.util.image_util import check_image_type -try: - eggs.require( "bx-python" ) - import bx.align.maf -except: - pass +import bx.align.maf log = logging.getLogger(__name__) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 65210229d55..6a6b390ce73 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -2,9 +2,6 @@ Custom exceptions for Galaxy """ -from galaxy import eggs -eggs.require( "Paste" ) - from paste import httpexceptions from ..exceptions import error_codes diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index da867d47aa6..8e5f8743bcb 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -8,8 +8,6 @@ import logging import socket from galaxy.util import send_mail from galaxy.util.json import dumps -from galaxy import eggs -eggs.require( "MarkupSafe" ) from markupsafe import escape log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/jobs/deferred/data_transfer.py b/lib/galaxy/jobs/deferred/data_transfer.py index 4589b2f7117..1b3019785db 100644 --- a/lib/galaxy/jobs/deferred/data_transfer.py +++ b/lib/galaxy/jobs/deferred/data_transfer.py @@ -3,7 +3,6 @@ Module for managing data transfer jobs. """ import logging, urllib2, re, shutil -from galaxy import eggs from sqlalchemy import and_ from galaxy.util.odict import odict diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 486a6cf223c..78789766ffe 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -12,7 +12,6 @@ import subprocess from Queue import Queue, Empty -import galaxy.eggs import galaxy.jobs from galaxy.jobs.command_factory import build_command from galaxy import model @@ -277,20 +276,12 @@ class BaseJobRunner( object ): external_metadata_proc.wait() log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id ) - def _get_egg_env_opts(self): - env = [] - crate = galaxy.eggs.Crate() - for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): - env.append('GALAXY_%s="%s"; export GALAXY_%s' % (opt.upper(), getattr(crate, opt), opt.upper())) - return env - def get_job_file(self, job_wrapper, **kwds): job_metrics = job_wrapper.app.job_metrics job_instrumenter = job_metrics.job_instrumenters[ job_wrapper.job_destination.id ] env_setup_commands = kwds.get( 'env_setup_commands', [] ) env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' ) - env_setup_commands.extend( self._get_egg_env_opts() or [] ) destination = job_wrapper.job_destination or {} envs = destination.get( "env", [] ) envs.extend( job_wrapper.environment_variables ) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 488ef93d8ec..c5f8f5fccbc 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -10,15 +10,12 @@ import subprocess import sys import time -from galaxy import eggs from galaxy import model from galaxy.jobs import JobDestination from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner from galaxy.util import asbool -eggs.require( "drmaa" ) - log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] diff --git a/lib/galaxy/jobs/runners/lwr_client/config_util.py b/lib/galaxy/jobs/runners/lwr_client/config_util.py index ab5453d0c8f..60d5f0d6d62 100644 --- a/lib/galaxy/jobs/runners/lwr_client/config_util.py +++ b/lib/galaxy/jobs/runners/lwr_client/config_util.py @@ -1,16 +1,8 @@ """ Generic interface for reading YAML/INI/JSON config files into nested dictionaries. """ -try: - from galaxy import eggs - eggs.require('PyYAML') -except Exception: - # If not in Galaxy, ignore this. - pass -try: - import yaml -except ImportError: - yaml = None +import yaml + try: from ConfigParser import ConfigParser except ImportError: diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index 651cdc11dc3..49c57fd6f49 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -13,23 +13,7 @@ from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner import pkg_resources -egg_message = """ - -The 'pbs' runner depends on 'pbs_python' which is not installed or not -configured properly. Galaxy's "scramble" system should make this installation -simple, please follow the instructions found at: - - http://wiki.galaxyproject.org/Admin/Config/Performance/Cluster - -Additional errors may follow: -%s -""" - -try: - pkg_resources.require( "pbs_python" ) - pbs = __import__( "pbs" ) -except Exception, e: - raise Exception( egg_message % str( e ) ) +import pbs log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 3e3a4af4b7c..ce64643320f 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -4,7 +4,6 @@ IPC with multiple process configurations. """ import os, subprocess, socket, logging, threading -from galaxy import eggs from galaxy.util import listify, json log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d666942bba4..894a2631e88 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,9 +5,6 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -from galaxy import eggs -eggs.require("pexpect") - import codecs import errno import logging diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 5387a582d90..c97a31d27d6 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -8,8 +8,6 @@ from sys import getsizeof from itertools import chain from collections import deque -from galaxy import eggs -eggs.require("SQLAlchemy") import sqlalchemy from galaxy import app diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index cd4bf721469..6c2d5a5111a 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -2,20 +2,10 @@ import sys import os.path import logging -from galaxy import eggs -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "decorator" ) # Required by sqlalchemy-migrate -eggs.require( "Tempita " ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy-migrate" ) - from sqlalchemy import * from sqlalchemy.exc import NoSuchTableError from migrate.versioning import repository, schema -from galaxy.model.orm import dialect_to_egg - log = logging.getLogger( __name__ ) # path relative to galaxy @@ -33,18 +23,6 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" """ - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - eggs.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) diff --git a/lib/galaxy/model/orm/__init__.py b/lib/galaxy/model/orm/__init__.py index f7470334cab..d6f0563020b 100644 --- a/lib/galaxy/model/orm/__init__.py +++ b/lib/galaxy/model/orm/__init__.py @@ -1,37 +1,3 @@ -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) - from sqlalchemy import * from sqlalchemy.orm import * from sqlalchemy.interfaces import * - -import logging -log = logging.getLogger( __name__ ) - - -dialect_to_egg = { - "sqlite": "pysqlite>=2", - "postgres": "psycopg2", - "postgresql": "psycopg2", - "mysql": "MySQL_python" -} - - -def load_egg_for_url( url ): - # Load the appropriate db module - dialect = __guess_dialect_for_url( url ) - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module's in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - - -def __guess_dialect_for_url( url ): - return (url.split(':', 1))[0] diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py index 7ddaafd4861..4183e985e54 100644 --- a/lib/galaxy/model/orm/engine_factory.py +++ b/lib/galaxy/model/orm/engine_factory.py @@ -2,12 +2,9 @@ import logging log = logging.getLogger( __name__ ) from sqlalchemy import create_engine -from galaxy.model.orm import load_egg_for_url def build_engine(url, engine_options, database_query_profiling_proxy=False, trace_logger=None): - load_egg_for_url( url ) - # Should we use the logging proxy? if database_query_profiling_proxy: import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index 4185791a613..1252376e551 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -4,17 +4,7 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...). import logging import os.path -from galaxy import eggs - -eggs.require( "decorator" ) -eggs.require( "Tempita" ) -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # required by sqlalchemy-migrate -eggs.require( "sqlalchemy_migrate" ) - from galaxy.util.properties import load_app_properties -from galaxy.model.orm import dialect_to_egg import pkg_resources @@ -48,21 +38,6 @@ DATABASE = { } -def require_dialect_egg( db_url ): - dialect = ( db_url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - - def read_config_file_arg( argv, default, old_default ): if '-c' in argv: pos = argv.index( '-c' ) @@ -130,5 +105,4 @@ def get_config( argv, cwd=None ): else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file - require_dialect_egg( db_url ) return dict(db_url=db_url, repo=repo, config_file=config_file, database=database) diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index ed8d10629e7..6812bbc7d25 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -27,8 +27,6 @@ select * from history where name='Unnamed history' import logging import re -from galaxy import eggs -eggs.require("Parsley") import parsley from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index e57fa2211c9..06e31eb8991 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -2,20 +2,10 @@ import sys import os.path import logging -from galaxy import eggs -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "decorator" ) # Required by sqlalchemy-migrate -eggs.require( "Tempita " ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy-migrate" ) - from sqlalchemy import * from sqlalchemy.exc import NoSuchTableError from migrate.versioning import repository, schema -from galaxy.model.orm import dialect_to_egg - log = logging.getLogger( __name__ ) # path relative to galaxy @@ -26,18 +16,6 @@ migrate_repository = repository.Repository( migrate_repository_directory ) def create_or_verify_database( url, engine_options={}, app=None ): """ """ - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - eggs.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4754156ad5b..a1c60f184fa 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -15,17 +15,7 @@ from posixpath import dirname as path_dirname from galaxy.exceptions import ObjectNotFound from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs -try: - import galaxy.eggs - galaxy.eggs.require( 'PyRods' ) -except Exception: - pass -try: - import irods -except ImportError: - irods = None - -NO_PYRODS_ERROR_MESSAGE = "IRODS object store configured, but no PyRods dependency available. Please install and properly configure PyRods or modify object store configuration." +import irods log = logging.getLogger( __name__ ) @@ -35,8 +25,6 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): Galaxy object store based on iRODS """ def __init__( self, config, file_path=None, extra_dirs=None ): - if irods is None: - raise Exception(NO_PYRODS_ERROR_MESSAGE) super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs ) self.cache_path = config.object_store_cache_path self.default_resource = config.irods_default_resource or None diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py index 623fdc8e439..a96161b8a88 100644 --- a/lib/galaxy/objectstore/s3_multipart_upload.py +++ b/lib/galaxy/objectstore/s3_multipart_upload.py @@ -14,17 +14,8 @@ import subprocess from multiprocessing.pool import IMapIterator -try: - from galaxy import eggs - eggs.require('boto') -except ImportError: - pass - -try: - import boto - from boto.s3.connection import S3Connection -except ImportError: - boto = None +import boto +from boto.s3.connection import S3Connection def map_wrap(f): diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 241567d41c6..6203b5a3c73 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -8,13 +8,7 @@ import threading import sys import galaxy.queues -from galaxy import eggs, util -eggs.require('anyjson') -if sys.version_info < (2, 7, 0): - # Kombu requires importlib and ordereddict to function under Python 2.6. - eggs.require('importlib') - eggs.require('ordereddict') -eggs.require('kombu') +from galaxy import util from kombu import Connection from kombu.mixins import ConsumerMixin diff --git a/lib/galaxy/queues.py b/lib/galaxy/queues.py index 6d5813d3faf..ae48b4d547e 100644 --- a/lib/galaxy/queues.py +++ b/lib/galaxy/queues.py @@ -6,16 +6,8 @@ All message queues used by Galaxy import sys -from galaxy import eggs from galaxy.config import process_is_uwsgi -eggs.require('PyYAML') -eggs.require('anyjson') -if sys.version_info < (2, 7, 0): - # Kombu requires importlib and ordereddict to function in Python 2.6. - eggs.require('importlib') - eggs.require('ordereddict') -eggs.require('kombu') from kombu import Exchange, Queue, Connection ALL_CONTROL = "control.*" diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 802bf6ca404..d271ab95502 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -12,13 +12,6 @@ import threading import urllib from datetime import datetime -from galaxy import eggs - -eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako -eggs.require( "Mako" ) -eggs.require( "Paste" ) -eggs.require( "SQLAlchemy >= 0.4" ) - from cgi import FieldStorage from xml.etree import ElementTree from mako.template import Template diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index 2a2b089bfbe..25c553c03d1 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -1,11 +1,5 @@ from __future__ import absolute_import -try: - from galaxy import eggs - eggs.require("PyYAML") -except ImportError: - pass - import yaml from .yaml import YamlToolSource diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 3293ec09c54..a4d5721bc4a 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -2,10 +2,8 @@ Module for building and searching the index of tools installed within this Galaxy. """ -from galaxy import eggs from galaxy.web.framework.helpers import to_unicode -eggs.require( "Whoosh" ) from whoosh.filedb.filestore import RamStorage from whoosh.fields import Schema, STORED, TEXT from whoosh.scoring import BM25F diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 1a04fdf4b74..17cc9ba4d1c 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -4,8 +4,6 @@ import string import tarfile import tempfile -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import and_ from galaxy.model.item_attrs import Dictifiable diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 39f84532aee..4778e8b9ac4 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -33,15 +33,11 @@ from os.path import relpath from hashlib import md5 from itertools import izip -from galaxy import eggs - -eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 from xml.etree import ElementTree, ElementInclude -eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English diff --git a/lib/galaxy/util/log/fluent_log.py b/lib/galaxy/util/log/fluent_log.py index 5806bb55a9b..36fb1ea39b0 100644 --- a/lib/galaxy/util/log/fluent_log.py +++ b/lib/galaxy/util/log/fluent_log.py @@ -5,23 +5,11 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector import time import threading -try: - import galaxy.eggs - galaxy.eggs.require( "fluent-logger" ) - galaxy.eggs.require( "msgpack_python" ) -except Exception: - pass - -try: - from fluent.sender import FluentSender -except ImportError: - FluentSender = None +from fluent.sender import FluentSender class FluentTraceLogger( object ): def __init__( self, name, host='localhost', port=24224 ): - if FluentSender is None: - raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.") self.lock = threading.Lock() self.thread_local = threading.local() self.name = name diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index b63d0c203d1..9b97d26004b 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -1,15 +1,6 @@ from xml.etree import ElementTree -try: - from galaxy import eggs - eggs.require('PyYAML') -except Exception: - # If not in Galaxy, ignore this. - pass -try: - import yaml -except ImportError: - yaml = None +import yaml from galaxy.util.submodules import submodules @@ -78,8 +69,5 @@ def plugin_source_from_path(path): def __read_yaml(path): - if yaml is None: - raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") - with open(path, "rb") as f: return yaml.load(f) diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 4a36d3f26f3..15ef9b7225e 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -9,13 +9,9 @@ import random import re import sys -from galaxy import eggs -eggs.require('numpy') # noqa -eggs.require('bx-python') # noqa from bx.interval_index_file import Indexes from bx.bbi.bigbed_file import BigBedFile from bx.bbi.bigwig_file import BigWigFile -eggs.require('pysam') # noqa from pysam import csamtools, ctabix from galaxy.datatypes.interval import Bed, Gff, Gtf diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 8ef748fbe7c..a44e749c94d 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -2,9 +2,5 @@ Galaxy web application framework """ -from galaxy import eggs -eggs.require( "pexpect" ) -eggs.require( "amqp" ) - import base url_for = base.routes.url_for diff --git a/lib/galaxy/web/framework/decorators.py b/lib/galaxy/web/framework/decorators.py index 00f1154c55d..4972c6b606b 100644 --- a/lib/galaxy/web/framework/decorators.py +++ b/lib/galaxy/web/framework/decorators.py @@ -2,8 +2,6 @@ import inspect from traceback import format_exc from functools import wraps -from galaxy import eggs -eggs.require( "Paste" ) import paste.httpexceptions from galaxy.web.framework import url_for diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index 89856277727..a6670587285 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -4,17 +4,12 @@ Galaxy web framework helpers import time from datetime import datetime, timedelta -from galaxy import eggs from galaxy.util import hash_util from galaxy.util.json import safe_dumps as dumps -eggs.require( "MarkupSafe" ) #required by WebHelpers -eggs.require( "WebHelpers" ) from markupsafe import escape from webhelpers import date from webhelpers.html.tags import stylesheet_link, javascript_link -eggs.require( "repoze.lru" ) # used by Routes -eggs.require( "Routes" ) from routes import url_for server_starttime = int(time.time()) diff --git a/lib/galaxy/web/framework/middleware/sentry.py b/lib/galaxy/web/framework/middleware/sentry.py index c81f39938cd..a9f1409550e 100644 --- a/lib/galaxy/web/framework/middleware/sentry.py +++ b/lib/galaxy/web/framework/middleware/sentry.py @@ -6,8 +6,6 @@ raven.middleware :license: BSD, see LICENSE for more details. """ -import galaxy.eggs; galaxy.eggs.require( "raven" ) - from raven import Client from raven.utils.wsgi import get_current_url, get_headers, \ get_environ diff --git a/lib/galaxy/web/framework/openid_manager.py b/lib/galaxy/web/framework/openid_manager.py index 6e3e5ec2f73..28580653746 100644 --- a/lib/galaxy/web/framework/openid_manager.py +++ b/lib/galaxy/web/framework/openid_manager.py @@ -4,9 +4,6 @@ Manage the OpenID consumer and related data stores. import os, pickle, logging -from galaxy import eggs -eggs.require( 'python-openid' ) - import openid from openid import oidutil from openid.store import filestore diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index e813c29672a..4da79e68995 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -10,18 +10,11 @@ import string import time from Cookie import CookieError -from galaxy import eggs -eggs.require( "Cheetah" ) from Cheetah.Template import Template -eggs.require( "Mako" ) import mako.runtime import mako.lookup -# pytz is used by Babel. -eggs.require( "pytz" ) -eggs.require( "Babel" ) from babel.support import Translations from babel import Locale -eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import and_ from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm import joinedload diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 88ceeaee72a..0948ce3ab1d 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -23,8 +23,6 @@ from galaxy import util from galaxy.util import asbool from galaxy.util.properties import load_app_properties -from galaxy import eggs -eggs.require('Paste') from paste import httpexceptions import logging @@ -663,7 +661,6 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. - eggs.require( "WebError" ) from weberror import evalexception app = evalexception.EvalException( app, conf, templating_formatters=build_template_error_formatters() ) diff --git a/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py b/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py index 59272b921e3..6bdca646056 100644 --- a/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py +++ b/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py @@ -10,16 +10,10 @@ import os import tempfile import time -from galaxy import eggs from galaxy import web from galaxy.web.base.controller import BaseUIController from galaxy.util.json import dumps -eggs.require('PyYAML') -eggs.require('boto') -eggs.require('simplejson') -eggs.require('bioblend') - from boto.exception import EC2ResponseError from bioblend import cloudman diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index e41bee9ea68..e30c1ffd83e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -2,7 +2,7 @@ import logging import os import urllib -from galaxy import datatypes, eggs, model, util, web +from galaxy import datatypes, model, util, web from galaxy import managers from galaxy.datatypes.display_applications.util import decode_dataset_user, encode_dataset_user from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings @@ -13,7 +13,6 @@ from galaxy.web.base.controller import BaseUIController, ERROR, SUCCESS, url_for from galaxy.web.framework.helpers import grids, iff, time_ago, to_unicode, escape from galaxy.tools.errors import EmailErrorReporter -eggs.require( "Paste" ) import paste.httpexceptions log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 2206c586baa..71010912e1a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -12,7 +12,6 @@ import urllib2 import zipfile from galaxy import util, web from galaxy.web import url_for -from galaxy.eggs import require from galaxy.security import Action from galaxy.tools.actions import upload_common from galaxy.util import inflector @@ -23,20 +22,14 @@ from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, bu from markupsafe import escape from galaxy.model.orm import and_, eagerload_all -# Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether tool search is enabled. -try: - require( "Whoosh" ) - import whoosh.index - from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT - from whoosh.scoring import BM25F - from whoosh.qparser import MultifieldParser - whoosh_search_enabled = True - # The following must be defined exactly like the - # schema in ~/scripts/data_libraries/build_whoosh_index.py - schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT ) -except ImportError, e: - whoosh_search_enabled = False - schema = None +import whoosh.index +from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT +from whoosh.scoring import BM25F +from whoosh.qparser import MultifieldParser +whoosh_search_enabled = True +# The following must be defined exactly like the +# schema in ~/scripts/data_libraries/build_whoosh_index.py +schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT ) log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 10665404401..be9a54adfa8 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -6,8 +6,6 @@ from galaxy.model.orm import * from galaxy import model, util from galaxy.web.form_builder import * from .requests_common import RequestsGrid, invalid_id_redirect -from galaxy import eggs -eggs.require("amqp") import amqp import logging, os, pexpect, ConfigParser diff --git a/lib/galaxy/webapps/reports/controllers/workflows.py b/lib/galaxy/webapps/reports/controllers/workflows.py index 562a1a55619..7e061898209 100644 --- a/lib/galaxy/webapps/reports/controllers/workflows.py +++ b/lib/galaxy/webapps/reports/controllers/workflows.py @@ -1,11 +1,9 @@ import calendar from datetime import date, timedelta -from galaxy import eggs from galaxy import model, util from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseUIController, web from galaxy.web.framework.helpers import grids -eggs.require( "SQLAlchemy >= 0.4" ) import sqlalchemy as sa from galaxy.webapps.reports.controllers.query import ReportQueryBuilder from galaxy.webapps.reports.controllers.jobs import sorter diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index da796d28e27..158cd40cb23 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -8,7 +8,6 @@ import logging import logging.config from optparse import OptionParser import ConfigParser -from galaxy import eggs from galaxy.util import string_as_bool, listify log = logging.getLogger( __name__ ) @@ -308,7 +307,6 @@ def configure_logging( config ): root.addHandler( handler ) # If sentry is configured, also log to it if config.sentry_dsn: - eggs.require( "raven" ) from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) diff --git a/lib/galaxy/webapps/tool_shed/controllers/hg.py b/lib/galaxy/webapps/tool_shed/controllers/hg.py index 48b850dceb6..57fab895b77 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -7,8 +7,6 @@ from tool_shed.util.shed_util_common import get_repository_by_name_and_owner from tool_shed.util.hg_util import update_repository from tool_shed.metadata import repository_metadata_manager -from galaxy import eggs -eggs.require('mercurial') import mercurial.__version__ from mercurial.hgweb.hgwebdir_mod import hgwebdir from mercurial.hgweb.request import wsgiapplication diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 188e35329b6..c58b9e43245 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -45,9 +45,6 @@ import tool_shed.grids.util as grids_util import tool_shed.repository_types.util as rt_util -from galaxy import eggs -eggs.require( 'mercurial' ) - from mercurial import mdiff from mercurial import patch diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index 10c6c409ec2..f0c0762deab 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -24,8 +24,6 @@ from tool_shed.util import shed_util_common as suc from tool_shed.util import repository_content_util from tool_shed.util import xml_util -from galaxy import eggs -eggs.require( 'mercurial' ) from mercurial import commands log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 6e2bfad7d73..8e6537d66cf 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -15,8 +15,6 @@ from tool_shed.util import hg_util from tool_shed.util import commit_util import tool_shed.repository_types.util as rt_util -from galaxy import eggs -eggs.require( 'mercurial' ) import mercurial.__version__ log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/tool_shed/model/__init__.py b/lib/galaxy/webapps/tool_shed/model/__init__.py index e13dceb7327..e3b7218d31e 100644 --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -9,8 +9,6 @@ from galaxy.util.hash_util import new_secure_hash from galaxy.model.item_attrs import Dictifiable import tool_shed.repository_types.util as rt_util -from galaxy import eggs -eggs.require( 'mercurial' ) from mercurial import hg from mercurial import ui diff --git a/lib/galaxy/webapps/tool_shed/model/migrate/check.py b/lib/galaxy/webapps/tool_shed/model/migrate/check.py index 4e87b2235e2..64f83133ced 100644 --- a/lib/galaxy/webapps/tool_shed/model/migrate/check.py +++ b/lib/galaxy/webapps/tool_shed/model/migrate/check.py @@ -2,22 +2,10 @@ import logging import os.path import sys -from galaxy import eggs - -eggs.require( "SQLAlchemy" ) -eggs.require( "decorator" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy-migrate" ) -eggs.require( "Tempita" ) - - from migrate.versioning import repository, schema from sqlalchemy import * from sqlalchemy.exc import NoSuchTableError -from galaxy.model.orm import dialect_to_egg - log = logging.getLogger( __name__ ) # path relative to galaxy @@ -36,18 +24,6 @@ def create_or_verify_database( url, engine_options={} ): 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" """ - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - eggs.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) meta = MetaData( bind=engine ) diff --git a/lib/galaxy/webapps/tool_shed/search/repo_search.py b/lib/galaxy/webapps/tool_shed/search/repo_search.py index 43320bbac8d..52e065126e5 100644 --- a/lib/galaxy/webapps/tool_shed/search/repo_search.py +++ b/lib/galaxy/webapps/tool_shed/search/repo_search.py @@ -1,13 +1,11 @@ """Module for searching the toolshed repositories""" import datetime from galaxy import exceptions -from galaxy import eggs from galaxy.webapps.tool_shed import model from galaxy.exceptions import ObjectNotFound import logging log = logging.getLogger( __name__ ) -eggs.require( "Whoosh" ) import whoosh.index from whoosh import scoring from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED diff --git a/lib/galaxy/webapps/tool_shed/search/tool_search.py b/lib/galaxy/webapps/tool_shed/search/tool_search.py index 19d324166e0..1e450c024f9 100644 --- a/lib/galaxy/webapps/tool_shed/search/tool_search.py +++ b/lib/galaxy/webapps/tool_shed/search/tool_search.py @@ -3,11 +3,9 @@ import datetime import os import logging from galaxy import exceptions -from galaxy import eggs from galaxy import web from galaxy.webapps.tool_shed import model from galaxy.exceptions import ObjectNotFound -eggs.require( "Whoosh" ) import whoosh.index from whoosh import scoring from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED diff --git a/lib/galaxy/workflow/render.py b/lib/galaxy/workflow/render.py index 739267fbea9..a719fa854d4 100644 --- a/lib/galaxy/workflow/render.py +++ b/lib/galaxy/workflow/render.py @@ -1,6 +1,3 @@ -from galaxy import eggs - -eggs.require( "SVGFig" ) import svgfig MARGIN = 5 diff --git a/lib/galaxy_ext/metadata/set_metadata.py b/lib/galaxy_ext/metadata/set_metadata.py index 8e1cf631eb7..f65f5714bf5 100644 --- a/lib/galaxy_ext/metadata/set_metadata.py +++ b/lib/galaxy_ext/metadata/set_metadata.py @@ -27,8 +27,6 @@ assert sys.version_info[:2] >= ( 2, 6 ) and sys.version_info[:2] <= ( 2, 7 ), 'P new_path = os.path.abspath( os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir ) ) sys.path.insert( 0, new_path ) -from galaxy import eggs -import pkg_resources import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers from galaxy.model.custom_types import total_size diff --git a/lib/pulsar/client/config_util.py b/lib/pulsar/client/config_util.py index ab5453d0c8f..e34c29afab7 100644 --- a/lib/pulsar/client/config_util.py +++ b/lib/pulsar/client/config_util.py @@ -1,16 +1,8 @@ """ Generic interface for reading YAML/INI/JSON config files into nested dictionaries. """ -try: - from galaxy import eggs - eggs.require('PyYAML') -except Exception: - # If not in Galaxy, ignore this. - pass -try: - import yaml -except ImportError: - yaml = None +import yaml + try: from ConfigParser import ConfigParser except ImportError: @@ -56,9 +48,6 @@ def __find_type(path, explicit_type, default_type): def __read_yaml(path): - if yaml is None: - raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") - with open(path, "rb") as f: return yaml.load(f) diff --git a/lib/pulsar/client/transport/poster.py b/lib/pulsar/client/transport/poster.py index 5b2ea4c815c..93086762e56 100644 --- a/lib/pulsar/client/transport/poster.py +++ b/lib/pulsar/client/transport/poster.py @@ -10,18 +10,8 @@ try: from urllib2 import Request except ImportError: from urllib.request import Request -try: - from galaxy import eggs - eggs.require("poster") -except ImportError: - pass -try: - import poster -except ImportError: - poster = None - -POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is unavailable. Please install poster." +import poster log = logging.getLogger(__name__) @@ -31,7 +21,6 @@ if poster is not None: def post_file(url, path): - __ensure_poster() try: datagen, headers = poster.encode.multipart_encode({"file": open(path, "rb")}) request = Request(url, datagen, headers) @@ -51,8 +40,3 @@ def get_file(url, path): if not buffer: break output.write(buffer) - - -def __ensure_poster(): - if poster is None: - raise ImportError(POSTER_UNAVAILABLE_MESSAGE) diff --git a/lib/pulsar/client/transport/requests.py b/lib/pulsar/client/transport/requests.py index e5ab4b33a22..ee3cec37532 100644 --- a/lib/pulsar/client/transport/requests.py +++ b/lib/pulsar/client/transport/requests.py @@ -2,16 +2,7 @@ from __future__ import absolute_import import logging -try: - from galaxy import eggs - eggs.require("requests") -except ImportError: - pass - -try: - import requests -except ImportError: - requests = None +import requests try: import requests_toolbelt @@ -21,7 +12,6 @@ except ImportError: requests_toolbelt = None -REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests." REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt." log = logging.getLogger(__name__) @@ -31,7 +21,6 @@ def post_file(url, path): if requests_toolbelt is None: raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE) - __ensure_requests() m = requests_toolbelt.MultipartEncoder( fields={'file': ('filename', open(path, 'rb'))} ) @@ -46,8 +35,3 @@ def get_file(url, path): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() - - -def __ensure_requests(): - if requests is None: - raise ImportError(REQUESTS_UNAVAILABLE_MESSAGE) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index e6ab0c03e5f..00e944d4108 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -6,13 +6,8 @@ import tempfile import traceback from galaxy import exceptions -from galaxy import eggs from galaxy import util -eggs.require( 'paramiko' ) -eggs.require( 'ssh' ) -eggs.require( 'Fabric' ) - from fabric.api import lcd from galaxy.model.orm import or_ diff --git a/lib/tool_shed/galaxy_install/migrate/check.py b/lib/tool_shed/galaxy_install/migrate/check.py index ced4bd6c6ba..018594e35ac 100644 --- a/lib/tool_shed/galaxy_install/migrate/check.py +++ b/lib/tool_shed/galaxy_install/migrate/check.py @@ -2,20 +2,12 @@ import logging import os import subprocess import sys -from galaxy import eggs -eggs.require( "decorator" ) -eggs.require( "Tempita" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "SQLAlchemy" ) -eggs.require( "sqlalchemy_migrate" ) from migrate.versioning import repository from migrate.versioning import schema from sqlalchemy import create_engine from sqlalchemy import MetaData from sqlalchemy import Table from galaxy.util.odict import odict -from galaxy.model.orm import dialect_to_egg from tool_shed.util import common_util log = logging.getLogger( __name__ ) @@ -27,18 +19,6 @@ migrate_repository = repository.Repository( migrate_repository_directory ) def verify_tools( app, url, galaxy_config_file, engine_options={} ): # Check the value in the migrate_tools.version database table column to verify that the number is in # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[ dialect ] - try: - eggs.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) meta = MetaData( bind=engine ) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py index d336c711f42..242f1075ebf 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -9,12 +9,6 @@ import time from contextlib import contextmanager # TODO: eliminate the use of fabric here. -from galaxy import eggs - -eggs.require( 'paramiko' ) -eggs.require( 'ssh' ) -eggs.require( 'Fabric' ) - from fabric.operations import _AttributeString from fabric import state from fabric.api import prefix diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index dc81364ee3e..5b6bd6f4864 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -18,12 +18,6 @@ from tool_shed.util import tool_dependency_util from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager # TODO: eliminate the use of fabric here. -from galaxy import eggs - -eggs.require( 'paramiko' ) -eggs.require( 'ssh' ) -eggs.require( 'Fabric' ) - from fabric.api import settings from fabric.api import lcd diff --git a/lib/tool_shed/grids/admin_grids.py b/lib/tool_shed/grids/admin_grids.py index 4489eadacdd..f82d6d525e5 100644 --- a/lib/tool_shed/grids/admin_grids.py +++ b/lib/tool_shed/grids/admin_grids.py @@ -9,9 +9,6 @@ from tool_shed.util import hg_util from tool_shed.grids.repository_grids import CategoryGrid from tool_shed.grids.repository_grids import RepositoryGrid -from galaxy import eggs -eggs.require( 'mercurial' ) - from mercurial import commands from mercurial import hg from mercurial import ui diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index d10427d5efe..07f176eb1bd 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -12,9 +12,6 @@ import tool_shed.grids.util as grids_util import tool_shed.repository_types.util as rt_util from tool_shed.util import metadata_util -from galaxy import eggs - -eggs.require('markupsafe') from markupsafe import escape as escape_html log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/repository_types/metadata.py b/lib/tool_shed/repository_types/metadata.py index 19442ba0451..cc066356981 100644 --- a/lib/tool_shed/repository_types/metadata.py +++ b/lib/tool_shed/repository_types/metadata.py @@ -1,8 +1,6 @@ import logging import tool_shed.repository_types.util as rt_util -from galaxy import eggs -eggs.require( 'mercurial' ) from mercurial import hg from mercurial import ui diff --git a/lib/tool_shed/repository_types/repository_suite_definition.py b/lib/tool_shed/repository_types/repository_suite_definition.py index 045c37b5d76..4e3aa31036d 100644 --- a/lib/tool_shed/repository_types/repository_suite_definition.py +++ b/lib/tool_shed/repository_types/repository_suite_definition.py @@ -3,8 +3,6 @@ from tool_shed.repository_types.metadata import TipOnly import tool_shed.repository_types.util as rt_util from tool_shed.util import basic_util -from galaxy import eggs -eggs.require( 'mercurial' ) from mercurial import hg from mercurial import ui diff --git a/lib/tool_shed/repository_types/tool_dependency_definition.py b/lib/tool_shed/repository_types/tool_dependency_definition.py index 760f5190ba6..287240ef56c 100644 --- a/lib/tool_shed/repository_types/tool_dependency_definition.py +++ b/lib/tool_shed/repository_types/tool_dependency_definition.py @@ -3,8 +3,6 @@ from tool_shed.repository_types.metadata import TipOnly import tool_shed.repository_types.util as rt_util from tool_shed.util import basic_util -from galaxy import eggs -eggs.require( 'mercurial' ) from mercurial import hg from mercurial import ui diff --git a/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py b/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py index d4f58307290..acd4594ebc7 100755 --- a/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py +++ b/lib/tool_shed/scripts/bootstrap_tool_shed/bootstrap_util.py @@ -8,8 +8,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) import galaxy.webapps.tool_shed.model.mapping as tool_shed_model from sqlalchemy.exc import ProgrammingError from sqlalchemy.exc import OperationalError diff --git a/lib/tool_shed/scripts/bootstrap_tool_shed/create_user_with_api_key.py b/lib/tool_shed/scripts/bootstrap_tool_shed/create_user_with_api_key.py index a15a9389d3c..90e79b2ac39 100644 --- a/lib/tool_shed/scripts/bootstrap_tool_shed/create_user_with_api_key.py +++ b/lib/tool_shed/scripts/bootstrap_tool_shed/create_user_with_api_key.py @@ -12,10 +12,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) -eggs.require( 'mercurial' ) - from galaxy.web import security import galaxy.webapps.tool_shed.config as tool_shed_config from galaxy.webapps.tool_shed.model import mapping diff --git a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py index c49d78f1f52..ebe19cdf1bb 100644 --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -8,10 +8,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ), new_path.extend( sys.path[ 1: ] ) sys.path = new_path -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) -eggs.require( 'mercurial' ) - import ConfigParser import galaxy.webapps.tool_shed.config as tool_shed_config from tool_shed.util import hg_util diff --git a/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py index f448331a03c..06986a04ba8 100644 --- a/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py +++ b/lib/tool_shed/scripts/check_s3_for_empty_tool_dependency_installation_paths.py @@ -7,8 +7,6 @@ new_path.extend( sys.path[1:] ) sys.path = new_path from galaxy.util import asbool -from galaxy import eggs -eggs.require( 'boto' ) import boto diff --git a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py index 58713bfd15b..93f06175291 100644 --- a/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py +++ b/lib/tool_shed/scripts/check_tool_dependency_definition_repositories.py @@ -16,10 +16,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ), os.path.join( os.getcwd(), "tes new_path.extend( sys.path[1:] ) sys.path = new_path -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) -eggs.require( 'mercurial' ) - from mercurial import __version__ import galaxy.webapps.tool_shed.config as tool_shed_config diff --git a/lib/tool_shed/scripts/show_tool_dependency_installation_dir_contents.py b/lib/tool_shed/scripts/show_tool_dependency_installation_dir_contents.py index 49433ad1687..ecc20a8ee90 100644 --- a/lib/tool_shed/scripts/show_tool_dependency_installation_dir_contents.py +++ b/lib/tool_shed/scripts/show_tool_dependency_installation_dir_contents.py @@ -6,9 +6,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[ 1: ] ) sys.path = new_path -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) - import galaxy.model import galaxy.model.tool_shed_install.mapping as install_mapper import galaxy.config as galaxy_config diff --git a/lib/tool_shed/util/basic_util.py b/lib/tool_shed/util/basic_util.py index 18497640206..b36950d6b98 100644 --- a/lib/tool_shed/util/basic_util.py +++ b/lib/tool_shed/util/basic_util.py @@ -6,9 +6,6 @@ from string import Template from galaxy.util import unicodify, nice_size -from galaxy import eggs - -eggs.require( 'MarkupSafe' ) import markupsafe log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py index f9f4d132137..4c04ac66934 100644 --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/tool_shed/util/encoding_util.py @@ -1,7 +1,6 @@ import binascii import json import logging -from galaxy import eggs from galaxy.util.hash_util import hmac_new from galaxy.util.json import json_fix diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index 1f46ab953d1..5793afb4021 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -9,8 +9,6 @@ from time import strftime import tempfile from galaxy.util import listify -from galaxy import eggs -eggs.require( 'mercurial' ) from mercurial import cmdutil from mercurial import commands From 431ebf93bd75437fb5f5c41e40791c5c9e6e9aad Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 15 Jul 2015 15:37:39 -0400 Subject: [PATCH 08/52] Updating requirements.txt as I get wheels ready. TODO: automate OS X wheels. --- requirements.txt | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9ee8016957e..17bc490759a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,16 @@ -# -# Galaxy has traditionally managed all of its own dependencies (in the form of -# precompiled Python eggs) for a few reasons: -# -# 1. We can ensure that Galaxy works with the versions of the dependencies that -# we require. -# 2. We can precompile them for numerous platforms, meaning that developers or -# system administrators attempting to use Galaxy do not need to wait while -# dependencies compile/install, nor do they need to have compilers or -# various development libraries installed on their system. -# -# However, you may want to manage these dependencies on your own. If so, this -# file can be used with pip to install all of the bare minimum dependencies for -# starting a Galaxy server. Use this in conjunction with the -# `GALAXY_ENABLE_EGGS=False` environment variable. -# -# Note that except in the case of Whoosh, all dependencies will be installed at -# their newest versions. Galaxy has not been tested with these and you may -# (likely?) run into problems! The Galaxy Team does not support using Galaxy -# with dependency versions other than those in eggs.ini, however, if you do -# find an incompatibility between Galaxy and the latest version, we would -# appreciate if you would investigate and provide a code fix for the issue. We -# are trying to move away from our strict version dependency policy and this -# would greatly help in our efforts. +# packages with C extensions +Cheetah=2.4.4 +bx-python==0.7.3 +MarkupSafe==0.23 +PyYAML==3.11 +SQLAlchemy==1.0.6 +mercurial==3.4.2 +numpy==1.9.2 +pycrypto==2.6.1 +# pure Python packages + +# TODO Paste PasteDeploy docutils @@ -32,19 +20,11 @@ amqp repoze.lru Routes WebOb -SQLAlchemy -Cheetah -pycrypto -MarkupSafe WebHelpers anyjson kombu -numpy -bx-python # bx must be installed after numpy! pysam Mako -PyYAML -mercurial pytz Babel decorator From 90279909209983fc620e26c6d7fabb6031d57dfd Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 13:20:48 -0400 Subject: [PATCH 09/52] Pin requirements --- requirements.txt | 85 +++++++++++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/requirements.txt b/requirements.txt index 17bc490759a..153e2bb8e85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # packages with C extensions -Cheetah=2.4.4 +Cheetah==2.4.4 bx-python==0.7.3 MarkupSafe==0.23 PyYAML==3.11 @@ -7,38 +7,57 @@ SQLAlchemy==1.0.6 mercurial==3.4.2 numpy==1.9.2 pycrypto==2.6.1 +python_lzo==1.8 +psycopg2==2.6.1 + +# extra functionality from Dannon +pysam==0.8.3gx1 + +# Optional (prebuilt) + +# Optional (DIY) +#MySQL_python +#PyRods +#pbs_python # pure Python packages +bioblend==0.6.1 +Paste==1.7.5.1 +PasteDeploy==1.5.2 +docutils==0.12 +wchartype==0.1 +amqp==1.4.6 +repoze.lru==0.6 +Routes==2.2 +WebOb==1.4.1 +WebHelpers==1.3 +anyjson==0.3.3 +kombu==3.0.26 +Mako==1.0.2 +pytz==2015.4 +Babel==2.0 +Beaker==1.7.0 -# TODO -Paste -PasteDeploy -docutils -wchartype -pexpect -amqp -repoze.lru -Routes -WebOb -WebHelpers -anyjson -kombu -pysam -Mako -pytz -Babel -decorator -Tempita -sqlalchemy-migrate -Beaker -Whoosh<2.5 -paramiko -ssh -Fabric -boto -simplejson -bioblend -Parsley -WebError -http://svgfig.googlecode.com/files/svgfig-1.1.6.tgz -nose +# sqlalchemy-migrate and dependencies +sqlalchemy-migrate==0.10.0 +decorator==4.0.2 +Tempita==0.5.3dev +sqlparse==0.1.16 +pbr==1.8.0 +six==1.9.0 +Parsley==1.3 +WebError==0.11 +nose==1.3.7 +SVGFig==1.1.6 + +# Fabric and dependencies +Fabric==1.10.2 +paramiko==1.15.2 +ecdsa==0.13 +# pycrypto, but it is a direct Galaxy dependency as well + +# Can't use Whoosh latest due to a bug but need to backport a bugfix from a +# newer release. +# +# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff +Whoosh==2.4.1gx1 From 40bde6e102ae7bd66973a9f81b79fa9a08836c55 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 13:21:13 -0400 Subject: [PATCH 10/52] Install required dependencies from requirements.txt rather than convoluted galaxy.wheels method. Not sure why I was doing that. Hackathons are dangerous places. --- lib/galaxy/wheels.py | 24 ++---------------------- scripts/common_startup.sh | 3 ++- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/wheels.py b/lib/galaxy/wheels.py index c94159779c4..ef5975500cb 100644 --- a/lib/galaxy/wheels.py +++ b/lib/galaxy/wheels.py @@ -3,31 +3,17 @@ import sys -from os import pardir -from os.path import abspath, join, dirname - try: import configparser except: import ConfigParser as configparser -BASE_REQUIREMENTS = [] OPTIONAL_REQUIREMENTS = ( 'pysqlite', 'psycopg2', ) -for line in open(join(abspath(dirname(__file__)), pardir, pardir, 'requirements.txt')): - line = line.strip() - try: - line = line.split()[0] - except: - continue - if line.startswith( '#' ) or line == '': - continue - BASE_REQUIREMENTS.append( line ) - class GalaxyConfig( object ): def __init__( self, config_file ): @@ -57,12 +43,6 @@ class GalaxyConfig( object ): return False -def requirements( config_file ): +def optional( config_file ): config = GalaxyConfig( config_file ) - rval = list(BASE_REQUIREMENTS) - - for opt in OPTIONAL_REQUIREMENTS: - if config.check(opt): - rval.append( opt ) - - return rval + return [ opt for opt in OPTIONAL_REQUIREMENTS if config.check( opt ) ] diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index f23c09aa2cb..fa55bcd06e6 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -95,5 +95,6 @@ fi if [ $FETCH_WHEELS -eq 1 ]; then pip install -e git+https://github.com/natefoo/pip@linux-wheels#egg=pip - PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.requirements('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin --index-url https://wheels.galaxyproject.org/simple/ + pip install -r requirements.txt --index-url https://wheels.galaxyproject.org/simple/ + PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.optional('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin --index-url https://wheels.galaxyproject.org/simple/ fi From 975fddeb4092097c5f6eb02f36cac65dcddf0d6a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 15:13:44 -0400 Subject: [PATCH 11/52] Fix linting errors --- lib/galaxy/datatypes/binary.py | 1 - .../datatypes/converters/vcf_to_vcf_bgzip.py | 1 - lib/galaxy/model/orm/scripts.py | 1 - lib/galaxy/queue_worker.py | 1 - lib/galaxy/queues.py | 2 -- lib/galaxy/wheels.py | 16 ++++++++-------- lib/pulsar/client/transport/poster.py | 13 ++++++++++++- lib/pulsar/client/transport/requests.py | 12 +++++++++++- 8 files changed, 31 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 127e6666119..ee20c164075 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -8,7 +8,6 @@ import shutil import struct import subprocess import tempfile -import warnings import zipfile from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py index d606fd51c10..02d42fe2964 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py +++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py @@ -7,7 +7,6 @@ Original ordering, which may be specifically needed by tools or external displa usage: %prog in_file out_file """ -import pkg_resources import ctabix import optparse diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index e408a4132f6..56258e002a4 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -6,7 +6,6 @@ import os.path from galaxy.util.properties import load_app_properties -import pkg_resources log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index cc7fadfadfd..3ed455cea01 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -5,7 +5,6 @@ reloading the toolbox, etc., across multiple processes. import logging import threading -import sys import galaxy.queues from galaxy import util diff --git a/lib/galaxy/queues.py b/lib/galaxy/queues.py index ae48b4d547e..0779c3dcb0d 100644 --- a/lib/galaxy/queues.py +++ b/lib/galaxy/queues.py @@ -4,8 +4,6 @@ All message queues used by Galaxy """ -import sys - from galaxy.config import process_is_uwsgi from kombu import Exchange, Queue, Connection diff --git a/lib/galaxy/wheels.py b/lib/galaxy/wheels.py index ef5975500cb..7390a1cfcb6 100644 --- a/lib/galaxy/wheels.py +++ b/lib/galaxy/wheels.py @@ -30,14 +30,14 @@ class GalaxyConfig( object ): return True else: try: - return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ), - "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), - "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), - "drmaa": lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), - "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), - "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), - "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), - "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" + return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ), + "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), + "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), + "drmaa": lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), + "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), + "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), + "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), + "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" }.get( name, lambda: True )() except: return False diff --git a/lib/pulsar/client/transport/poster.py b/lib/pulsar/client/transport/poster.py index 93086762e56..f8ea3f7d9d3 100644 --- a/lib/pulsar/client/transport/poster.py +++ b/lib/pulsar/client/transport/poster.py @@ -11,7 +11,12 @@ try: except ImportError: from urllib.request import Request -import poster +try: + import poster +except ImportError: + poster = None + +POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is unavailable. Please install poster." log = logging.getLogger(__name__) @@ -21,6 +26,7 @@ if poster is not None: def post_file(url, path): + __ensure_poster() try: datagen, headers = poster.encode.multipart_encode({"file": open(path, "rb")}) request = Request(url, datagen, headers) @@ -40,3 +46,8 @@ def get_file(url, path): if not buffer: break output.write(buffer) + + +def __ensure_poster(): + if poster is None: + raise ImportError(POSTER_UNAVAILABLE_MESSAGE) diff --git a/lib/pulsar/client/transport/requests.py b/lib/pulsar/client/transport/requests.py index ee3cec37532..1e5c0c4ff97 100644 --- a/lib/pulsar/client/transport/requests.py +++ b/lib/pulsar/client/transport/requests.py @@ -2,7 +2,10 @@ from __future__ import absolute_import import logging -import requests +try: + import requests +except ImportError: + requests = None try: import requests_toolbelt @@ -12,6 +15,7 @@ except ImportError: requests_toolbelt = None +REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests." REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt." log = logging.getLogger(__name__) @@ -21,6 +25,7 @@ def post_file(url, path): if requests_toolbelt is None: raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE) + __ensure_requests() m = requests_toolbelt.MultipartEncoder( fields={'file': ('filename', open(path, 'rb'))} ) @@ -35,3 +40,8 @@ def get_file(url, path): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() + + +def __ensure_requests(): + if requests is None: + raise ImportError(REQUESTS_UNAVAILABLE_MESSAGE) From 0469ed29e064a4aed213dcc0230e8a06e711fc24 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 15:55:07 -0400 Subject: [PATCH 12/52] Remove egg cruft from scripts/ --- scripts/api/common.py | 7 - scripts/check_galaxy.py | 2 - .../admin_cleanup_datasets.py | 3 - scripts/cleanup_datasets/cleanup_datasets.py | 10 +- scripts/cleanup_datasets/pgcleanup.py | 3 - scripts/data_libraries/build_lucene_index.py | 3 - scripts/data_libraries/build_whoosh_index.py | 3 - scripts/db_shell.py | 4 - scripts/drmaa_external_killer.py | 6 - scripts/drmaa_external_runner.py | 6 - scripts/egg_packager_template.py | 245 ------------------ scripts/functional_tests.py | 12 - scripts/manage_db.py | 9 - scripts/manage_tools.py | 12 - scripts/nosetests.py | 8 - scripts/others/incorrect_gops_jobs.py | 5 +- scripts/others/incorrect_gops_join_jobs.py | 5 +- scripts/paster.py | 5 - scripts/summarize_timings.py | 9 - scripts/tool_shed/build_ts_whoosh_index.py | 3 - .../migrate_tools_to_repositories.py | 3 - scripts/tools/maf/check_loc_file.py | 2 - scripts/transfer.py | 9 +- scripts/update_shed_config_path.py | 21 -- 24 files changed, 7 insertions(+), 388 deletions(-) delete mode 100644 scripts/egg_packager_template.py diff --git a/scripts/api/common.py b/scripts/api/common.py index 1380b7fe31b..9767baf633a 100644 --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -7,13 +7,6 @@ import os import sys import urllib2 -new_path = [ os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, 'lib' ) ] -new_path.extend( sys.path[1:] ) -sys.path = new_path - -from galaxy import eggs -eggs.require( "pycrypto" ) - from Crypto.Cipher import Blowfish log = logging.getLogger( __name__ ) diff --git a/scripts/check_galaxy.py b/scripts/check_galaxy.py index 20ba2ae4627..16d51d18a32 100755 --- a/scripts/check_galaxy.py +++ b/scripts/check_galaxy.py @@ -97,8 +97,6 @@ If the user does not exist, check_galaxy will create it for you.""" % login_file # find/import twill lib_dir = os.path.join( scripts_dir, os.pardir, "lib" ) sys.path.insert( 1, lib_dir ) -from galaxy import eggs -eggs.require( "twill" ) import twill import twill.commands as tc diff --git a/scripts/cleanup_datasets/admin_cleanup_datasets.py b/scripts/cleanup_datasets/admin_cleanup_datasets.py index c66016f37dc..0d9f3b8da49 100755 --- a/scripts/cleanup_datasets/admin_cleanup_datasets.py +++ b/scripts/cleanup_datasets/admin_cleanup_datasets.py @@ -49,11 +49,8 @@ from time import strftime from cleanup_datasets import CleanupDatasetsApplication -from galaxy import eggs -eggs.require('SQLAlchemy') import sqlalchemy as sa from sqlalchemy import and_, false -eggs.require("Mako") from mako.template import Template import galaxy.config diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index e189ef11ca2..3eee234bc88 100755 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -10,16 +10,14 @@ from datetime import datetime, timedelta from optparse import OptionParser from time import strftime -new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path -sys.path = new_path - -from galaxy import eggs -eggs.require('SQLAlchemy') import sqlalchemy as sa from sqlalchemy import and_, false, null, true from sqlalchemy.orm import eagerload +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) # remove scripts/ from the path +sys.path = new_path + import galaxy.config import galaxy.model.mapping from galaxy.exceptions import ObjectNotFound diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py index 780d984a23c..7dfa428368e 100755 --- a/scripts/cleanup_datasets/pgcleanup.py +++ b/scripts/cleanup_datasets/pgcleanup.py @@ -17,10 +17,7 @@ from optparse import OptionParser galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) sys.path.insert(0, os.path.join(galaxy_root, 'lib')) -from galaxy import eggs -eggs.require('psycopg2') import psycopg2 -eggs.require('SQLAlchemy') from sqlalchemy.engine.url import make_url import galaxy.config diff --git a/scripts/data_libraries/build_lucene_index.py b/scripts/data_libraries/build_lucene_index.py index 902cfcf445e..a49ea5689ee 100644 --- a/scripts/data_libraries/build_lucene_index.py +++ b/scripts/data_libraries/build_lucene_index.py @@ -20,11 +20,8 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs import galaxy.model.mapping from galaxy import config, model -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) def main( ini_file ): sa_session, gconfig = get_sa_session( ini_file ) diff --git a/scripts/data_libraries/build_whoosh_index.py b/scripts/data_libraries/build_whoosh_index.py index f04fc8d3ffd..bcc395534b7 100644 --- a/scripts/data_libraries/build_whoosh_index.py +++ b/scripts/data_libraries/build_whoosh_index.py @@ -14,12 +14,9 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs # Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether search is enabled. try: - eggs.require( "Whoosh" ) - from whoosh.filedb.filestore import FileStorage from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT from whoosh.index import Index diff --git a/scripts/db_shell.py b/scripts/db_shell.py index 2bb25101ae2..5e3d8c5e0a9 100644 --- a/scripts/db_shell.py +++ b/scripts/db_shell.py @@ -22,10 +22,6 @@ sys.path = new_path from galaxy.model.orm.scripts import get_config -from galaxy import eggs -eggs.require( "decorator" ) -eggs.require( "Tempita" ) -eggs.require( "SQLAlchemy" ) db_url = get_config( sys.argv )['db_url'] diff --git a/scripts/drmaa_external_killer.py b/scripts/drmaa_external_killer.py index 02419bc34c1..6c74a0cfaab 100755 --- a/scripts/drmaa_external_killer.py +++ b/scripts/drmaa_external_killer.py @@ -9,12 +9,6 @@ import os import pwd import sys -new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path -sys.path = new_path - -from galaxy import eggs -eggs.require("drmaa") import drmaa diff --git a/scripts/drmaa_external_runner.py b/scripts/drmaa_external_runner.py index 6b4d04c8403..a1474feb418 100755 --- a/scripts/drmaa_external_runner.py +++ b/scripts/drmaa_external_runner.py @@ -12,12 +12,6 @@ import os import pwd import sys -new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path -sys.path = new_path - -from galaxy import eggs -eggs.require("drmaa") import drmaa DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', diff --git a/scripts/egg_packager_template.py b/scripts/egg_packager_template.py deleted file mode 100644 index 478b92b569d..00000000000 --- a/scripts/egg_packager_template.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python - -# Configure stdout logging - -import glob -import HTMLParser -import logging -import os -import re -import shutil -import sys -import urllib -import urllib2 -import zipfile - -log = logging.getLogger() -log.setLevel( 10 ) -log.addHandler( logging.StreamHandler( sys.stdout ) ) - -# Fake pkg_resources - -macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)") -darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") -solarisVersionString = re.compile(r"solaris-(\d)\.(\d+)-(.*)") - - -def compatible_platforms(provided, required): - """Can code for the `provided` platform run on the `required` platform? - - Returns true if either platform is ``None``, or the platforms are equal. - - XXX Needs compatibility checks for Linux and other unixy OSes. - """ - if provided is None or required is None or provided == required: - return True # easy case - - # Mac OS X special cases - reqMac = macosVersionString.match(required) - if reqMac: - provMac = macosVersionString.match(provided) - - # is this a Mac package? - if not provMac: - # this is backwards compatibility for packages built before - # setuptools 0.6. All packages built after this point will - # use the new macosx designation. - provDarwin = darwinVersionString.match(provided) - if provDarwin: - dversion = int(provDarwin.group(1)) - macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2)) - if dversion == 7 and macosversion >= "10.3" or \ - dversion == 8 and macosversion >= "10.4": - - # import warnings - # warnings.warn("Mac eggs should be rebuilt to " - # "use the macosx designation instead of darwin.", - # category=DeprecationWarning) - return True - return False # egg isn't macosx or legacy darwin - - # are they the same major version and machine type? - if provMac.group(1) != reqMac.group(1) or \ - provMac.group(3) != reqMac.group(3): - return False - - # is the required OS major update >= the provided one? - if int(provMac.group(2)) > int(reqMac.group(2)): - return False - - return True - - # Solaris' special cases - reqSol = solarisVersionString.match(required) - if reqSol: - provSol = solarisVersionString.match(provided) - if not provSol: - return False - if provSol.group(1) != reqSol.group(1) or \ - provSol.group(3) != reqSol.group(3): - return False - if int(provSol.group(2)) > int(reqSol.group(2)): - return False - return True - - # XXX Linux and other platforms' special cases should go here - return False - -EGG_NAME = re.compile( - r"(?P[^-]+)" - r"( -(?P[^-]+) (-py(?P[^-]+) (-(?P.+))? )? )?", - re.VERBOSE | re.IGNORECASE -).match - - -class Distribution( object ): - def __init__( self, egg_name, project_name, version, py_version, platform ): - self._egg_name = egg_name - self.project_name = project_name - if project_name is not None: - self.project_name = project_name.replace( '-', '_' ) - self.version = version - if version is not None: - self.version = version.replace( '-', '_' ) - self.py_version = py_version - self.platform = platform - self.location = os.path.join( tmpd, egg_name ) + '.egg' - - def egg_name( self ): - return self._egg_name - - @classmethod - def from_filename( cls, basename ): - project_name, version, py_version, platform = [None] * 4 - basename, ext = os.path.splitext(basename) - if ext.lower() == '.egg': - match = EGG_NAME( basename ) - if match: - project_name, version, py_version, platform = match.group( 'name', 'ver', 'pyver', 'plat' ) - return cls( basename, project_name, version, py_version, platform ) - - -class pkg_resources( object ): - pass - -pkg_resources.Distribution = Distribution - -# Fake galaxy.eggs - -env = None - - -def get_env(): - return None - - -class URLRetriever( urllib.FancyURLopener ): - def http_error_default( *args ): - urllib.URLopener.http_error_default( *args ) - - -class Egg( object ): - def __init__( self, distribution ): - self.url = url + '/' + distribution.project_name.replace( '-', '_' ) - self.dir = tmpd - self.distribution = distribution - - def set_distribution( self ): - pass - - def unpack_if_needed( self ): - pass - - def remove_doppelgangers( self ): - pass - - def fetch( self, requirement ): - """ - fetch() serves as the install method to pkg_resources.working_set.resolve() - """ - def find_alternative(): - """ - Some platforms (e.g. Solaris) support eggs compiled on older platforms - """ - class LinkParser( HTMLParser.HTMLParser ): - """ - Finds links in what should be an Apache-style directory index - """ - def __init__( self ): - HTMLParser.HTMLParser.__init__( self ) - self.links = [] - - def handle_starttag( self, tag, attrs ): - if tag == 'a' and 'href' in dict( attrs ): - self.links.append( dict( attrs )['href'] ) - parser = LinkParser() - try: - parser.feed( urllib2.urlopen( self.url + '/' ).read() ) - except urllib2.HTTPError, e: - if e.code == 404: - return None - parser.close() - for link in parser.links: - file = urllib.unquote( link ).rsplit( '/', 1 )[-1] - tmp_dist = pkg_resources.Distribution.from_filename( file ) - if tmp_dist.platform is not None and \ - self.distribution.project_name == tmp_dist.project_name and \ - self.distribution.version == tmp_dist.version and \ - self.distribution.py_version == tmp_dist.py_version and \ - compatible_platforms( tmp_dist.platform, self.distribution.platform ): - return file - return None - if self.url is None: - return None - alternative = None - try: - url = self.url + '/' + self.distribution.egg_name() + '.egg' - URLRetriever().retrieve( url, self.distribution.location ) - log.debug( "Fetched %s" % url ) - except IOError, e: - if e[1] == 404 and self.distribution.platform != py: - alternative = find_alternative() - if alternative is None: - return None - else: - return None - if alternative is not None: - try: - url = '/'.join( ( self.url, alternative ) ) - URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) ) - log.debug( "Fetched %s" % url ) - except IOError, e: - return None - self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0] - self.set_distribution() - self.unpack_if_needed() - self.remove_doppelgangers() - global env - env = get_env() # reset the global Environment object now that we've obtained a new egg - return self.distribution - - -def create_zip(): - fname = 'galaxy_eggs-%s.zip' % platform - z = zipfile.ZipFile( fname, 'w', zipfile.ZIP_STORED ) - for egg in glob.glob( os.path.join( tmpd, '*.egg' ) ): - z.write( egg, 'eggs/' + os.path.basename( egg ) ) - z.close() - print 'Egg package is in %s' % fname - print "To install the eggs, please copy this file to your Galaxy installation's root" - print "directory and unpack with:" - print " unzip %s" % fname - - -def clean(): - shutil.rmtree( tmpd ) - -import tempfile -tmpd = tempfile.mkdtemp() - -failures = [] -platform = None -py = None -url = None - -# Automatically generated egg definitions follow diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index e25a885ddbd..f698e9fe2d7 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -16,20 +16,8 @@ sys.path = new_path from base.test_logging import logging_config_file from base.tool_shed_util import parse_tool_panel_config -from galaxy import eggs from galaxy.util.properties import load_app_properties -eggs.require( "nose" ) -eggs.require( "NoseHTML" ) -eggs.require( "NoseTestDiff" ) -eggs.require( "Paste" ) -eggs.require( "PasteDeploy" ) -eggs.require( "Cheetah" ) - -# this should not be required, but it is under certain conditions, thanks to this bug: -# http://code.google.com/p/python-nose/issues/detail?id=284 -eggs.require( "pysqlite" ) - import logging import os.path import time diff --git a/scripts/manage_db.py b/scripts/manage_db.py index 98e504927fa..bce683b8e31 100644 --- a/scripts/manage_db.py +++ b/scripts/manage_db.py @@ -8,15 +8,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs - -eggs.require( "decorator" ) -eggs.require( "Tempita" ) -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy_migrate" ) - from migrate.versioning.shell import main from galaxy.model.orm.scripts import get_config diff --git a/scripts/manage_tools.py b/scripts/manage_tools.py index 46ca456415f..b6776eb8a0a 100644 --- a/scripts/manage_tools.py +++ b/scripts/manage_tools.py @@ -6,19 +6,9 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs - -eggs.require( "SQLAlchemy" ) -eggs.require( "decorator" ) -eggs.require( "Tempita " ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy-migrate" ) - from migrate.versioning.shell import main from ConfigParser import SafeConfigParser -from galaxy.model.orm.scripts import require_dialect_egg from galaxy.model.orm.scripts import read_config_file_arg from galaxy.util.properties import load_app_properties @@ -46,6 +36,4 @@ elif "database_file" in properties: else: db_url = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE" -require_dialect_egg( db_url ) - main( repository=repo, url=db_url ) diff --git a/scripts/nosetests.py b/scripts/nosetests.py index 41f5e8b10e0..1ffebd70343 100755 --- a/scripts/nosetests.py +++ b/scripts/nosetests.py @@ -4,14 +4,6 @@ import os import sys -new_path = [ os.path.join( os.getcwd(), "lib" ) ] -new_path.extend( sys.path[1:] ) -sys.path = new_path - -from galaxy import eggs -eggs.require( 'nose' ) -eggs.require( 'NoseHTML' ) - from pkg_resources import load_entry_point assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/scripts/others/incorrect_gops_jobs.py b/scripts/others/incorrect_gops_jobs.py index 7553778b3f1..da0782d268e 100755 --- a/scripts/others/incorrect_gops_jobs.py +++ b/scripts/others/incorrect_gops_jobs.py @@ -4,13 +4,10 @@ Fetch jobs using gops_intersect, gops_merge, gops_subtract, gops_complement, gop wherein the second dataset doesn't have chr, start and end in standard columns 1, 2 and 3. """ -from galaxy import eggs import sys, os, ConfigParser, tempfile import galaxy.app import galaxy.model.mapping -import pkg_resources - -pkg_resources.require( "SQLAlchemy >= 0.4" ) + import sqlalchemy as sa assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/scripts/others/incorrect_gops_join_jobs.py b/scripts/others/incorrect_gops_join_jobs.py index 88c0779f8df..8290cf9894e 100644 --- a/scripts/others/incorrect_gops_join_jobs.py +++ b/scripts/others/incorrect_gops_join_jobs.py @@ -3,13 +3,10 @@ Fetch gops_join wherein the use specified minimum coverage is not 1. """ -from galaxy import eggs import sys, os, ConfigParser, tempfile import galaxy.app import galaxy.model.mapping -import pkg_resources - -pkg_resources.require( "SQLAlchemy >= 0.4" ) + import sqlalchemy as sa assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/scripts/paster.py b/scripts/paster.py index 9e7dd96ede7..85bb3ef3421 100755 --- a/scripts/paster.py +++ b/scripts/paster.py @@ -19,14 +19,9 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs - if 'LOG_TEMPFILES' in os.environ: from log_tempfile import TempFile _log_tempfile = TempFile() -eggs.require( "Paste" ) -eggs.require( "PasteDeploy" ) - from galaxy.util.pastescript import serve serve.run() diff --git a/scripts/summarize_timings.py b/scripts/summarize_timings.py index d5b768babc1..5ff7359c467 100644 --- a/scripts/summarize_timings.py +++ b/scripts/summarize_timings.py @@ -3,21 +3,12 @@ from __future__ import print_function import os import sys -new_path = [ os.path.join( os.path.dirname(__file__), os.pardir, "lib" ) ] -new_path.extend( sys.path[1:] ) # remove scripts/ from the path -sys.path = new_path - try: from argparse import ArgumentParser except ImportError: ArgumentParser = None import re -try: - from galaxy import eggs - eggs.require("numpy") -except ImportError: - pass import numpy diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index 776ad310a2b..d523cd24aa0 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -26,9 +26,6 @@ from galaxy.webapps.tool_shed import config from galaxy.webapps.tool_shed import model from galaxy.tools.loader_directory import load_tool_elements_from_path -from galaxy import eggs -eggs.require( "SQLAlchemy" ) -eggs.require( "Whoosh" ) from whoosh.filedb.filestore import FileStorage from whoosh.fields import Schema, STORED, TEXT diff --git a/scripts/tool_shed/migrate_tools_to_repositories.py b/scripts/tool_shed/migrate_tools_to_repositories.py index b1a51698877..d10e359d391 100644 --- a/scripts/tool_shed/migrate_tools_to_repositories.py +++ b/scripts/tool_shed/migrate_tools_to_repositories.py @@ -27,9 +27,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -from galaxy import eggs -import pkg_resources -pkg_resources.require( "psycopg2" ) import psycopg2 import galaxy.webapps.tool_shed.app diff --git a/scripts/tools/maf/check_loc_file.py b/scripts/tools/maf/check_loc_file.py index 1c41c3a314d..96514962db6 100644 --- a/scripts/tools/maf/check_loc_file.py +++ b/scripts/tools/maf/check_loc_file.py @@ -1,8 +1,6 @@ #Dan Blankenberg #This script checks maf_index.loc file for inconsistencies between what is listed as available and what is really available. #Make sure that required dependencies (e.g. galaxy_root/lib and galaxy_root/eggs) are included in your PYTHONPATH -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) import bx.align.maf from galaxy.tools.util import maf_utilities import sys diff --git a/scripts/transfer.py b/scripts/transfer.py index 7074b1b81dc..e194d2bd7fa 100644 --- a/scripts/transfer.py +++ b/scripts/transfer.py @@ -9,24 +9,18 @@ import os, sys, optparse, ConfigParser, socket, SocketServer, threading, logging galaxy_root = os.path.abspath( os.path.join( os.path.dirname( __file__ ), os.pardir ) ) sys.path.insert( 0, os.path.abspath( os.path.join( galaxy_root, 'lib' ) ) ) -from galaxy import eggs - try: import pexpect except ImportError: pexpect = None -eggs.require( "SQLAlchemy >= 0.4" ) - from sqlalchemy import * from sqlalchemy.orm import * -from galaxy.model.mapping import load_egg_for_url +from daemon import DaemonContext import galaxy.model from galaxy.util import json, bunch -eggs.require( 'python_daemon' ) -from daemon import DaemonContext PEXPECT_IMPORT_MESSAGE = ('The Python pexpect package is required to use this ' 'feature, please install it') @@ -88,7 +82,6 @@ class GalaxyApp( object ): dburl = self.config.get( 'app:main', 'database_connection' ) except ConfigParser.NoOptionError: dburl = default_dburl - load_egg_for_url( dburl ) engine = create_engine( dburl ) metadata = MetaData( engine ) self.sa_session = scoped_session( sessionmaker( bind=engine, autoflush=False, autocommit=True ) ) diff --git a/scripts/update_shed_config_path.py b/scripts/update_shed_config_path.py index a840b3b2c4f..895a75fd8d4 100644 --- a/scripts/update_shed_config_path.py +++ b/scripts/update_shed_config_path.py @@ -6,8 +6,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import create_engine, MetaData from sqlalchemy.orm import scoped_session, sessionmaker @@ -41,25 +39,6 @@ def create_database( config_file ): exit(1) '''Initialize the database file.''' - dialect_to_egg = { - "sqlite" : "pysqlite>=2", - "postgres" : "psycopg2", - "postgresql" : "psycopg2", - "mysql" : "MySQL_python" - } - dialect = ( database_connection.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[ dialect ] - try: - eggs.require( egg ) - print( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - print( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support. - print( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - # Initialize the database connection. engine = create_engine( database_connection ) MetaData( bind=engine ) From b727174326292fcc713adc79471065d483ce42a8 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 15:55:30 -0400 Subject: [PATCH 13/52] tox: Install nose and NoseHTML for unit tests. --- tox.ini | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 54521a66993..1b94d1a2fed 100644 --- a/tox.ini +++ b/tox.ini @@ -15,11 +15,17 @@ deps = flake8 [testenv:py27-unit] commands = bash run_tests.sh -u whitelist_externals = bash +deps = + nose + NoseHTML [testenv:py26-unit] commands = bash run_tests.sh -u whitelist_externals = bash -deps = unittest2 +deps = + unittest2 + nose + NoseHTML [testenv:qunit] commands = bash run_tests.sh -q From 93410ecf2538e32bdac7bed45367dbd6c993fb04 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 16:01:22 -0400 Subject: [PATCH 14/52] Fix additional lint errors created by modifying stuff in scripts/ --- scripts/nosetests.py | 1 - scripts/summarize_timings.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/scripts/nosetests.py b/scripts/nosetests.py index 1ffebd70343..2a2e2256630 100755 --- a/scripts/nosetests.py +++ b/scripts/nosetests.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # EASY-INSTALL-ENTRY-SCRIPT: 'nose','console_scripts','nosetests' # __requires__ = 'nose' -import os import sys from pkg_resources import load_entry_point diff --git a/scripts/summarize_timings.py b/scripts/summarize_timings.py index 5ff7359c467..e4e9a6f422a 100644 --- a/scripts/summarize_timings.py +++ b/scripts/summarize_timings.py @@ -1,8 +1,5 @@ from __future__ import print_function -import os -import sys - try: from argparse import ArgumentParser except ImportError: From 165d6bdc9ded10c751e4eecbcca4390d231875ee Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 16:07:25 -0400 Subject: [PATCH 15/52] Fix ctabix import errors (would these converters ever have worked?) --- lib/galaxy/datatypes/converters/bgzip.py | 3 ++- lib/galaxy/datatypes/converters/interval_to_tabix_converter.py | 3 ++- lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/converters/bgzip.py b/lib/galaxy/datatypes/converters/bgzip.py index 52176bc3162..2eb5c063292 100644 --- a/lib/galaxy/datatypes/converters/bgzip.py +++ b/lib/galaxy/datatypes/converters/bgzip.py @@ -6,11 +6,12 @@ Uses pysam to bgzip a file usage: %prog in_file out_file """ -import ctabix import tempfile import optparse import subprocess +from pysam import ctabix + def main(): # Read options, args. diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py index f598066a4a2..a36e6ae7f9d 100644 --- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py @@ -7,11 +7,12 @@ Supported presets: bed, gff, vcf usage: %prog in_file out_file """ -import ctabix import sys import os import optparse +from pysam import ctabix + def main(): # Read options, args. diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py index 02d42fe2964..eebf5cc5278 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py +++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py @@ -7,9 +7,10 @@ Original ordering, which may be specifically needed by tools or external displa usage: %prog in_file out_file """ -import ctabix import optparse +from pysam import ctabix + def main(): # Read options, args. From 222bb4294a3dd9c9c58ae9f0785a10eb361c1e77 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 21 Sep 2015 16:07:49 -0400 Subject: [PATCH 16/52] Conditional import for irods --- lib/galaxy/objectstore/rods.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index a1c60f184fa..5567953b699 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -15,7 +15,14 @@ from posixpath import dirname as path_dirname from galaxy.exceptions import ObjectNotFound from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs -import irods +try: + import irods +except ImportError: + irods = None + + +IRODS_IMPORT_MESSAGE = ('The Python irods package is required to use this ' + 'feature, please install it') log = logging.getLogger( __name__ ) @@ -26,6 +33,7 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): """ def __init__( self, config, file_path=None, extra_dirs=None ): super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs ) + assert irods is not None, IRODS_IMPORT_MESSAGE self.cache_path = config.object_store_cache_path self.default_resource = config.irods_default_resource or None From bf50d44ee50f5ace7d34db3e69c39779283167c7 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Sep 2015 10:31:41 -0400 Subject: [PATCH 17/52] Conditional imports for optional dependencies. --- lib/galaxy/util/log/fluent_log.py | 10 +++++++- lib/galaxy/web/framework/middleware/sentry.py | 12 ++++++++-- lib/galaxy/web/framework/openid_manager.py | 23 +++++++++++++++---- lib/galaxy/wheels.py | 2 +- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/util/log/fluent_log.py b/lib/galaxy/util/log/fluent_log.py index ba8ad4d4f79..e1e6224e74a 100644 --- a/lib/galaxy/util/log/fluent_log.py +++ b/lib/galaxy/util/log/fluent_log.py @@ -4,11 +4,19 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector import threading -from fluent.sender import FluentSender +try: + from fluent.sender import FluentSender +except ImportError: + FluentSender = None + + +FLUENT_IMPORT_MESSAGE = ('The Python fluent package is required to use this ' + 'feature, please install it') class FluentTraceLogger( object ): def __init__( self, name, host='localhost', port=24224 ): + assert FluentSender is not None, FLUENT_IMPORT_MESSAGE self.lock = threading.Lock() self.thread_local = threading.local() self.name = name diff --git a/lib/galaxy/web/framework/middleware/sentry.py b/lib/galaxy/web/framework/middleware/sentry.py index 34ea7e8ee45..6470ce9a91c 100644 --- a/lib/galaxy/web/framework/middleware/sentry.py +++ b/lib/galaxy/web/framework/middleware/sentry.py @@ -6,8 +6,15 @@ raven.middleware :license: BSD, see LICENSE for more details. """ -from raven import Client -from raven.utils.wsgi import get_current_url, get_headers, get_environ +try: + from raven import Client + from raven.utils.wsgi import get_current_url, get_headers, get_environ +except: + Client = None + + +RAVEN_IMPORT_MESSAGE = ('The Python raven package is required to use this ' + 'feature, please install it') class Sentry(object): @@ -16,6 +23,7 @@ class Sentry(object): uncaught exceptions and send them to Sentry. """ def __init__(self, application, dsn): + assert Client is not None, RAVEN_IMPORT_MESSAGE self.application = application self.client = Client( dsn ) diff --git a/lib/galaxy/web/framework/openid_manager.py b/lib/galaxy/web/framework/openid_manager.py index 291600cbb71..ee18db6f8fa 100644 --- a/lib/galaxy/web/framework/openid_manager.py +++ b/lib/galaxy/web/framework/openid_manager.py @@ -6,21 +6,34 @@ import os import pickle import logging -from openid import oidutil -from openid.store import filestore -from openid.consumer import consumer -from openid.extensions import sreg +try: + from openid import oidutil + from openid.store import filestore + from openid.consumer import consumer + from openid.extensions import sreg +except ImportError: + oidutil = None + + class FakeConsumer( object ): + __getattr__ = lambda x, y: None + consumer = FakeConsumer() + + +OPENID_IMPORT_MESSAGE = ('The Python openid package is required to use this ' + 'feature, please install it') log = logging.getLogger( __name__ ) def oidlog( message, level=0 ): log.debug( message ) -oidutil.log = oidlog +if oidutil is not None: + oidutil.log = oidlog class OpenIDManager( object ): def __init__( self, cache_path ): + assert oidutil is not None, OPENID_IMPORT_MESSAGE self.session_path = os.path.join( cache_path, 'session' ) self.store_path = os.path.join( cache_path, 'store' ) for dir in self.session_path, self.store_path: diff --git a/lib/galaxy/wheels.py b/lib/galaxy/wheels.py index 7390a1cfcb6..455423b1b14 100644 --- a/lib/galaxy/wheels.py +++ b/lib/galaxy/wheels.py @@ -35,7 +35,7 @@ class GalaxyConfig( object ): "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), "drmaa": lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), - "python_openid": lambda: self.config.get( "app:main", "enable_openid" ), + "openid": lambda: self.config.get( "app:main", "enable_openid" ), "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" }.get( name, lambda: True )() From f33476462722ec17947522d0a88661c88b688438 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Sep 2015 10:32:16 -0400 Subject: [PATCH 18/52] Egg cleaning in test/ --- test/base/instrument.py | 7 ------- test/base/interactor.py | 2 -- test/base/nose_util.py | 5 ----- test/base/test_db_util.py | 2 -- test/base/twilltestcase.py | 3 --- test/docker/base/Dockerfile | 3 +-- test/docker/base/ansible_vars.yml | 1 - test/functional/test_metadata_editing.py | 2 -- .../base/test_db_util.py | 2 -- .../base/twilltestcase.py | 2 -- .../base/util.py | 3 --- .../repositories_with_tools/functional_tests.py | 7 ------- .../tool_dependency_definitions/functional_tests.py | 6 ------ test/manual/workflows_scaling.py | 6 ------ test/tool_shed/base/test_db_util.py | 2 -- test/tool_shed/base/twilltestcase.py | 3 --- test/tool_shed/functional_tests.py | 11 ----------- test/unit/managers/base.py | 2 -- test/unit/managers/test_DatasetManager.py | 2 -- test/unit/managers/test_HDAManager.py | 2 -- test/unit/managers/test_HistoryManager.py | 2 -- test/unit/managers/test_UserManager.py | 2 -- test/unit/tools/test_execution.py | 2 -- test/unit/workflows/test_modules.py | 4 +--- tox.ini | 2 ++ 25 files changed, 4 insertions(+), 81 deletions(-) diff --git a/test/base/instrument.py b/test/base/instrument.py index dc7d92b9958..a5155276ed4 100644 --- a/test/base/instrument.py +++ b/test/base/instrument.py @@ -9,13 +9,6 @@ types in a different way. import json import threading -try: - from galaxy import eggs - eggs.require( "nose" ) -except ImportError: - pass - - from nose.plugins import Plugin NO_JOB_DATA = object() diff --git a/test/base/interactor.py b/test/base/interactor.py index 590fb18a441..8521f875d6b 100644 --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -4,8 +4,6 @@ from json import dumps from logging import getLogger from StringIO import StringIO -from galaxy import eggs -eggs.require( "requests" ) from requests import get, post from galaxy import util diff --git a/test/base/nose_util.py b/test/base/nose_util.py index 82a547c642b..67d792a72d7 100644 --- a/test/base/nose_util.py +++ b/test/base/nose_util.py @@ -4,11 +4,6 @@ There was some duplication between Galaxy, Tool Shed, and Install/Test, trying to reduce that here. """ -try: - from galaxy import eggs - eggs.require( "nose" ) -except ImportError: - pass import nose diff --git a/test/base/test_db_util.py b/test/base/test_db_util.py index 49b0c0f023d..4332072eb80 100644 --- a/test/base/test_db_util.py +++ b/test/base/test_db_util.py @@ -1,5 +1,3 @@ -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, desc, false import galaxy.model diff --git a/test/base/twilltestcase.py b/test/base/twilltestcase.py index 88fdae40c6b..d9af8ffccb4 100644 --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -17,10 +17,7 @@ from json import loads from urlparse import urlparse from xml.etree import ElementTree -from galaxy import eggs -eggs.require( "MarkupSafe" ) from markupsafe import escape -eggs.require( 'twill' ) import twill import twill.commands as tc from twill.other_packages._mechanize_dist import ClientForm diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 1a735a169f0..ef1b23ba61e 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -33,8 +33,7 @@ ADD start_mysql.sh /opt/galaxy/start_mysql.sh ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml ADD provision.yml /tmp/ansible/provision.yml -ENV PYTHON_EGG_CACHE=/root/.python-eggs \ - GALAXY_ROOT=/galaxy +ENV GALAXY_ROOT=/galaxy RUN mkdir /etc/galaxy && \ cd /tmp/ansible && \ diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml index e39108ac433..61f275b5e71 100644 --- a/test/docker/base/ansible_vars.yml +++ b/test/docker/base/ansible_vars.yml @@ -4,7 +4,6 @@ galaxyFS_base_dir: /opt/galaxy # TODO: use GALAXY_ROOT via cmd line instead galaxy_server_dir: "/galaxy" galaxy_venv_dir: "{{ galaxyFS_base_dir }}/.venv" -galxy_egg_cache: "/root/.python-eggs" galaxy_repo: https://bitbucket.org/galaxy/galaxy-central galaxy_changeset_id: stable galaxy_manage_clone: no diff --git a/test/functional/test_metadata_editing.py b/test/functional/test_metadata_editing.py index c6a3db6ea83..948bf2e3ba2 100644 --- a/test/functional/test_metadata_editing.py +++ b/test/functional/test_metadata_editing.py @@ -1,5 +1,3 @@ -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, desc, false import galaxy.model diff --git a/test/install_and_test_tool_shed_repositories/base/test_db_util.py b/test/install_and_test_tool_shed_repositories/base/test_db_util.py index 2c62fa3cde6..587f4c30ae7 100644 --- a/test/install_and_test_tool_shed_repositories/base/test_db_util.py +++ b/test/install_and_test_tool_shed_repositories/base/test_db_util.py @@ -1,7 +1,5 @@ import logging -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_ import galaxy.model as model diff --git a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py index 40e9bb16c12..58ee1904ff4 100644 --- a/test/install_and_test_tool_shed_repositories/base/twilltestcase.py +++ b/test/install_and_test_tool_shed_repositories/base/twilltestcase.py @@ -3,8 +3,6 @@ import os import re import time -from galaxy import eggs -eggs.require( 'twill' ) import twill.commands as tc import galaxy.model.tool_shed_install as install_model diff --git a/test/install_and_test_tool_shed_repositories/base/util.py b/test/install_and_test_tool_shed_repositories/base/util.py index c70d4d606f0..e0ee09460d1 100644 --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -13,9 +13,6 @@ sys.path[1:1] = [ os.path.join( galaxy_root, "lib" ), os.path.join( galaxy_root, 'test' ), os.path.join( galaxy_root, 'lib', 'tool_shed', 'scripts', 'api' ) ] -from galaxy import eggs -eggs.require( 'mercurial' ) -eggs.require( "nose" ) import nose from mercurial import hg, ui from nose.plugins import Plugin diff --git a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py index e59af2f4452..52238c72cbb 100644 --- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py @@ -22,13 +22,6 @@ sys.path[1:1] = [ os.path.join( galaxy_root, "scripts" ), os.path.join( galaxy_root, 'test' ), os.path.join( galaxy_root, 'scripts', 'api' ) ] -from galaxy import eggs -eggs.require( "nose" ) -eggs.require( "Paste" ) -eggs.require( 'mercurial' ) -# This should not be required, but it is under certain conditions thanks to this bug: -# http://code.google.com/p/python-nose/issues/detail?id=284 -eggs.require( "pysqlite" ) import nose from paste import httpserver diff --git a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py index f5aa03974c9..259fc17c3ea 100644 --- a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py @@ -21,12 +21,6 @@ sys.path[1:1] = [ os.path.join( galaxy_root, "scripts" ), os.path.join( galaxy_root, 'test' ), os.path.join( galaxy_root, 'scripts', 'api' ) ] -from galaxy import eggs -eggs.require( "Paste" ) -eggs.require( 'mercurial' ) -# This should not be required, but it is under certain conditions thanks to this bug: -# http://code.google.com/p/python-nose/issues/detail?id=284 -eggs.require( "pysqlite" ) from paste import httpserver import install_and_test_tool_shed_repositories.base.util as install_and_test_base_util diff --git a/test/manual/workflows_scaling.py b/test/manual/workflows_scaling.py index e71e45605c3..13c55eef9b7 100644 --- a/test/manual/workflows_scaling.py +++ b/test/manual/workflows_scaling.py @@ -14,12 +14,6 @@ try: except ImportError: ArgumentParser = None -try: - from galaxy import eggs - eggs.require("requests") - eggs.require("bioblend") -except ImportError: - pass import requests from bioblend import galaxy diff --git a/test/tool_shed/base/test_db_util.py b/test/tool_shed/base/test_db_util.py index 59716209b51..a2f4240fad3 100644 --- a/test/tool_shed/base/test_db_util.py +++ b/test/tool_shed/base/test_db_util.py @@ -1,7 +1,5 @@ import logging -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, false, true import galaxy.model diff --git a/test/tool_shed/base/twilltestcase.py b/test/tool_shed/base/twilltestcase.py index c970569709f..c69c2265283 100644 --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -8,9 +8,6 @@ import tempfile import time import urllib -from galaxy import eggs -eggs.require( 'mercurial' ) -eggs.require( 'twill' ) import twill.commands as tc from mercurial import commands, hg, ui from mercurial.util import Abort diff --git a/test/tool_shed/functional_tests.py b/test/tool_shed/functional_tests.py index a585f84b905..6d3da018b51 100644 --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -25,17 +25,6 @@ os.environ[ 'TOOL_SHED_TEST_TMP_DIR' ] = tool_shed_test_tmp_dir # Need to remove this directory from sys.path sys.path[0:1] = [ os.path.join( galaxy_root, "lib" ), os.path.join( galaxy_root, "test" ) ] -from galaxy import eggs -eggs.require( "nose" ) -eggs.require( "NoseHTML" ) -eggs.require( "NoseTestDiff" ) -eggs.require( "Paste" ) -eggs.require( "PasteDeploy" ) -eggs.require( "Cheetah" ) - -# This should not be required, but it is under certain conditions, thanks to this bug: http://code.google.com/p/python-nose/issues/detail?id=284 -eggs.require( "pysqlite" ) - from paste import httpserver import nose.config import nose.core diff --git a/test/unit/managers/base.py b/test/unit/managers/base.py index b51f7976a2b..1df3cda3a9b 100644 --- a/test/unit/managers/base.py +++ b/test/unit/managers/base.py @@ -11,8 +11,6 @@ test_utils = imp.load_source( 'test_utils', os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) ) import galaxy_mock -from galaxy import eggs -eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy.managers.users import UserManager diff --git a/test/unit/managers/test_DatasetManager.py b/test/unit/managers/test_DatasetManager.py index 9405a42680c..5b1349995e5 100644 --- a/test/unit/managers/test_DatasetManager.py +++ b/test/unit/managers/test_DatasetManager.py @@ -8,8 +8,6 @@ import unittest test_utils = imp.load_source( 'test_utils', os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) ) -from galaxy import eggs -eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy import model diff --git a/test/unit/managers/test_HDAManager.py b/test/unit/managers/test_HDAManager.py index 7362667006d..8ac45d1771a 100644 --- a/test/unit/managers/test_HDAManager.py +++ b/test/unit/managers/test_HDAManager.py @@ -6,8 +6,6 @@ import unittest test_utils = imp.load_source( 'test_utils', os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) ) -from galaxy import eggs -eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy import model diff --git a/test/unit/managers/test_HistoryManager.py b/test/unit/managers/test_HistoryManager.py index e5254d8e106..1cf64e47d70 100644 --- a/test/unit/managers/test_HistoryManager.py +++ b/test/unit/managers/test_HistoryManager.py @@ -10,8 +10,6 @@ test_utils = imp.load_source( 'test_utils', os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) ) import galaxy_mock -from galaxy import eggs -eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from sqlalchemy import true diff --git a/test/unit/managers/test_UserManager.py b/test/unit/managers/test_UserManager.py index 878709dcb6e..8f37b6fba76 100644 --- a/test/unit/managers/test_UserManager.py +++ b/test/unit/managers/test_UserManager.py @@ -8,8 +8,6 @@ import unittest test_utils = imp.load_source( 'test_utils', os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) ) -from galaxy import eggs -eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy import model diff --git a/test/unit/tools/test_execution.py b/test/unit/tools/test_execution.py index cab7989e85b..9a919a21b56 100644 --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -12,8 +12,6 @@ from galaxy.util import object_to_string from galaxy.util.odict import odict import tools_support -from galaxy import eggs -eggs.require( "Paste" ) from paste import httpexceptions BASE_REPEAT_TOOL_CONTENTS = ''' diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py index 3f728d34fb3..4a92b716d23 100644 --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -1,7 +1,5 @@ -from galaxy import eggs -eggs.require( "mock" ) - import json + import mock from galaxy import model diff --git a/tox.ini b/tox.ini index 1b94d1a2fed..3aa17c11e1f 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ whitelist_externals = bash deps = nose NoseHTML + mock [testenv:py26-unit] commands = bash run_tests.sh -u @@ -26,6 +27,7 @@ deps = unittest2 nose NoseHTML + mock [testenv:qunit] commands = bash run_tests.sh -q From a44b2c2170c9c326a2d5e1fec35beb44fd9834df Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Sep 2015 12:54:03 -0400 Subject: [PATCH 19/52] Rename galaxy.wheels to the more generalized galaxy.dependencies (since technically we don't care whether they are wheels), handle optional deps more correctly. Add error handling to common_startup.sh. --- lib/galaxy/dependencies/__init__.py | 91 +++++++++++++++++++ .../dependencies/conditional_requirements.txt | 9 ++ lib/galaxy/dependencies/requirements.txt | 57 ++++++++++++ lib/galaxy/wheels.py | 48 ---------- requirements.txt | 64 +------------ run.sh | 2 +- scripts/common_startup.sh | 6 +- 7 files changed, 162 insertions(+), 115 deletions(-) create mode 100644 lib/galaxy/dependencies/__init__.py create mode 100644 lib/galaxy/dependencies/conditional_requirements.txt create mode 100644 lib/galaxy/dependencies/requirements.txt delete mode 100644 lib/galaxy/wheels.py mode change 100644 => 120000 requirements.txt diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py new file mode 100644 index 00000000000..a65f64185c0 --- /dev/null +++ b/lib/galaxy/dependencies/__init__.py @@ -0,0 +1,91 @@ +""" +Determine what optional dependencies are needed. +""" + +try: + import configparser +except: + import ConfigParser as configparser + +import pkg_resources + +from os.path import dirname, join +from xml.etree import ElementTree + +from galaxy.util import asbool + + +class ConditionalDependencies( object ): + def __init__( self, config_file ): + self.config_file = config_file + self.config = None + self.job_runners = [] + self.conditional_reqs = [] + self.parse_configs() + self.get_conditional_requirements() + + def parse_configs( self ): + self.config = configparser.ConfigParser() + if not self.config.read( self.config_file ): + raise Exception( "Unable to read Galaxy config from %s" % self.config_file ) + try: + job_conf_xml = self.config.get( "app:main", "job_config_file" ) + except configparser.NoOptionError: + job_conf_xml = join( dirname( self.config_file ), 'config/job_conf.xml') + try: + for plugin in ElementTree.parse( job_conf_xml ).find( 'plugins' ): + self.job_runners.append( plugin.attrib.load ) + except (OSError, IOError): + pass + + def get_conditional_requirements( self ): + crfile = join( dirname( __file__ ), 'conditional_requirements.txt' ) + for req in pkg_resources.parse_requirements( open( crfile ).readlines() ): + self.conditional_reqs.append( req ) + + def check( self, name ): + try: + name = name.replace('-', '_').replace('.', '_') + return getattr( self, 'check_' + name )() + except: + return False + + def check_pysqlite( self ): + try: + return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) + except: + return True + + def check_psycopg2( self ): + return self.config.get( "app:main", "database_connection" ).startswith( "postgres" ) + + def check_mysql_python( self ): + return self.config.get( "app:main", "database_connection" ).startswith( "mysql" ) + + def check_drmaa( self ): + return "galaxy.jobs.runners.drmaa:DRMAAJobRunner" in self.job_runners + + def check_pbs( self ): + return "galaxy.jobs.runners.pbs:PBSJobRunner" in self.job_runners + + def check_python_openid( self ): + return asbool( self.config.get( "app:main", "enable_openid" ) ) + + def check_fluent_logger( self ): + return asbool( self.config.get( "app:main", "fluent_log" ) ) + + def check_raven( self ): + return asbool( self.config.get( "app:main", "sentry_dsn" ) ) + + def check_weberror( self ): + return ( asbool( self.config.get( "app:main", "debug" ) ) and + asbool( self.config_get( "app:main", "use_interactive" ) ) ) + + +def optional( config_file ): + rval = [] + conditional = ConditionalDependencies( config_file ) + for opt in conditional.conditional_reqs: + if conditional.check( opt.key ): + rval.append( str( opt ) ) + return rval diff --git a/lib/galaxy/dependencies/conditional_requirements.txt b/lib/galaxy/dependencies/conditional_requirements.txt new file mode 100644 index 00000000000..f539cd11391 --- /dev/null +++ b/lib/galaxy/dependencies/conditional_requirements.txt @@ -0,0 +1,9 @@ +# These dependencies are only required when certain config options are set +psycopg2==2.6.1 +pysqlite==2.6.3 +WebError==0.11 +python-openid +MySQL-python +fluent-logger +raven +# PyRods not in PyPI diff --git a/lib/galaxy/dependencies/requirements.txt b/lib/galaxy/dependencies/requirements.txt new file mode 100644 index 00000000000..355f1a978c5 --- /dev/null +++ b/lib/galaxy/dependencies/requirements.txt @@ -0,0 +1,57 @@ +# packages with C extensions +Cheetah==2.4.4 +bx-python==0.7.3 +MarkupSafe==0.23 +PyYAML==3.11 +SQLAlchemy==1.0.6 +mercurial==3.4.2 +numpy==1.9.2 +pycrypto==2.6.1 +python_lzo==1.8 +psycopg2==2.6.1 + +# extra functionality from Dannon +pysam==0.8.3gx1 + +# pure Python packages +bioblend==0.6.1 +Paste==1.7.5.1 +PasteDeploy==1.5.2 +docutils==0.12 +wchartype==0.1 +repoze.lru==0.6 +Routes==2.2 +WebOb==1.4.1 +WebHelpers==1.3 +Mako==1.0.2 +pytz==2015.4 +Babel==2.0 +Beaker==1.7.0 + +# kombu and dependencies +kombu==3.0.26 +amqp==1.4.6 +anyjson==0.3.3 + +# sqlalchemy-migrate and dependencies +sqlalchemy-migrate==0.10.0 +decorator==4.0.2 +Tempita==0.5.3dev +sqlparse==0.1.16 +pbr==1.8.0 +six==1.9.0 # six is also a Pulsar client dep +Parsley==1.3 +nose==1.3.7 +SVGFig==1.1.6 + +# Fabric and dependencies +Fabric==1.10.2 +paramiko==1.15.2 +ecdsa==0.13 +# pycrypto, but it is a direct Galaxy dependency as well + +# Can't use Whoosh latest due to a bug but need to backport a bugfix from a +# newer release. +# +# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff +Whoosh==2.4.1gx1 diff --git a/lib/galaxy/wheels.py b/lib/galaxy/wheels.py deleted file mode 100644 index 455423b1b14..00000000000 --- a/lib/galaxy/wheels.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -""" - -import sys - -try: - import configparser -except: - import ConfigParser as configparser - - -OPTIONAL_REQUIREMENTS = ( - 'pysqlite', - 'psycopg2', -) - - -class GalaxyConfig( object ): - def __init__( self, config_file ): - self.config = configparser.ConfigParser() - if self.config.read( config_file ) == []: - raise Exception( "error: unable to read Galaxy config from %s" % config_file ) - - def check( self, name ): - # SQLite is different since it can be specified in two config vars and defaults to True - if name == "pysqlite": - try: - return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) - except: - return True - else: - try: - return { "psycopg2": lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ), - "MySQL_python": lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), - "DRMAA_python": lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), - "drmaa": lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), - "pbs_python": lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), - "openid": lambda: self.config.get( "app:main", "enable_openid" ), - "python_daemon": lambda: sys.version_info[:2] >= ( 2, 5 ), - "PyRods": lambda: self.config.get( "app:main", "object_store" ) == "irods" - }.get( name, lambda: True )() - except: - return False - - -def optional( config_file ): - config = GalaxyConfig( config_file ) - return [ opt for opt in OPTIONAL_REQUIREMENTS if config.check( opt ) ] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 153e2bb8e85..00000000000 --- a/requirements.txt +++ /dev/null @@ -1,63 +0,0 @@ -# packages with C extensions -Cheetah==2.4.4 -bx-python==0.7.3 -MarkupSafe==0.23 -PyYAML==3.11 -SQLAlchemy==1.0.6 -mercurial==3.4.2 -numpy==1.9.2 -pycrypto==2.6.1 -python_lzo==1.8 -psycopg2==2.6.1 - -# extra functionality from Dannon -pysam==0.8.3gx1 - -# Optional (prebuilt) - -# Optional (DIY) -#MySQL_python -#PyRods -#pbs_python - -# pure Python packages -bioblend==0.6.1 -Paste==1.7.5.1 -PasteDeploy==1.5.2 -docutils==0.12 -wchartype==0.1 -amqp==1.4.6 -repoze.lru==0.6 -Routes==2.2 -WebOb==1.4.1 -WebHelpers==1.3 -anyjson==0.3.3 -kombu==3.0.26 -Mako==1.0.2 -pytz==2015.4 -Babel==2.0 -Beaker==1.7.0 - -# sqlalchemy-migrate and dependencies -sqlalchemy-migrate==0.10.0 -decorator==4.0.2 -Tempita==0.5.3dev -sqlparse==0.1.16 -pbr==1.8.0 -six==1.9.0 -Parsley==1.3 -WebError==0.11 -nose==1.3.7 -SVGFig==1.1.6 - -# Fabric and dependencies -Fabric==1.10.2 -paramiko==1.15.2 -ecdsa==0.13 -# pycrypto, but it is a direct Galaxy dependency as well - -# Can't use Whoosh latest due to a bug but need to backport a bugfix from a -# newer release. -# -# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff -Whoosh==2.4.1gx1 diff --git a/requirements.txt b/requirements.txt new file mode 120000 index 00000000000..2ea49591169 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +lib/galaxy/dependencies/requirements.txt \ No newline at end of file diff --git a/run.sh b/run.sh index ebc66407c5e..c8e19998df1 100755 --- a/run.sh +++ b/run.sh @@ -17,7 +17,7 @@ fi python ./scripts/check_python.py [ $? -ne 0 ] && exit 1 -./scripts/common_startup.sh +./scripts/common_startup.sh || exit 1 # If there is a .venv/ directory, assume it contains a virtualenv that we # should run this instance in. diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index fa55bcd06e6..28096d7df56 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -94,7 +94,7 @@ fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install -e git+https://github.com/natefoo/pip@linux-wheels#egg=pip - pip install -r requirements.txt --index-url https://wheels.galaxyproject.org/simple/ - PYTHONPATH=lib python -c "import galaxy.wheels; print '\n'.join(galaxy.wheels.optional('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin --index-url https://wheels.galaxyproject.org/simple/ + pip install -e git+https://github.com/natefoo/pip@linux-wheels#egg=pip && + pip install -r requirements.txt --index-url https://wheels.galaxyproject.org/simple/ && + PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))" | pip install -r /dev/stdin --index-url https://wheels.galaxyproject.org/simple/ || exit 1 fi From c0cb01c2f65ccf94e96120b080eb7b5aa06bf31b Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Sep 2015 13:51:48 -0400 Subject: [PATCH 20/52] Fixes for job_conf.xml conditional dependencies --- lib/galaxy/dependencies/__init__.py | 6 +++--- lib/galaxy/dependencies/conditional_requirements.txt | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index a65f64185c0..2366d0a1f52 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -31,10 +31,10 @@ class ConditionalDependencies( object ): try: job_conf_xml = self.config.get( "app:main", "job_config_file" ) except configparser.NoOptionError: - job_conf_xml = join( dirname( self.config_file ), 'config/job_conf.xml') + job_conf_xml = join( dirname( self.config_file ), 'job_conf.xml') try: for plugin in ElementTree.parse( job_conf_xml ).find( 'plugins' ): - self.job_runners.append( plugin.attrib.load ) + self.job_runners.append( plugin.attrib['load'] ) except (OSError, IOError): pass @@ -65,7 +65,7 @@ class ConditionalDependencies( object ): def check_drmaa( self ): return "galaxy.jobs.runners.drmaa:DRMAAJobRunner" in self.job_runners - def check_pbs( self ): + def check_pbs_python( self ): return "galaxy.jobs.runners.pbs:PBSJobRunner" in self.job_runners def check_python_openid( self ): diff --git a/lib/galaxy/dependencies/conditional_requirements.txt b/lib/galaxy/dependencies/conditional_requirements.txt index f539cd11391..dbd9b68deda 100644 --- a/lib/galaxy/dependencies/conditional_requirements.txt +++ b/lib/galaxy/dependencies/conditional_requirements.txt @@ -6,4 +6,6 @@ python-openid MySQL-python fluent-logger raven +pbs_python +drmaa # PyRods not in PyPI From e6337269b6fcd2b6c9d76045ff068a22d1dc39f8 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Sep 2015 13:52:14 -0400 Subject: [PATCH 21/52] Remove pkg_resources.requires from lib/ --- .../datatypes/display_applications/util.py | 2 -- lib/galaxy/datatypes/interval.py | 5 ----- lib/galaxy/datatypes/tabular.py | 2 -- lib/galaxy/datatypes/util/gff_util.py | 2 -- lib/galaxy/jobs/runners/drmaa.py | 20 +++++++++++-------- lib/galaxy/jobs/runners/pbs.py | 14 ++++++++++--- lib/galaxy/managers/base.py | 5 ----- lib/galaxy/managers/citations.py | 3 --- lib/galaxy/managers/histories.py | 2 -- lib/galaxy/managers/users.py | 2 -- lib/galaxy/tools/util/maf_utilities.py | 3 +-- lib/galaxy/util/template.py | 3 --- lib/galaxy/visualization/plugins/plugin.py | 3 --- lib/galaxy/web/base/controller.py | 3 --- lib/galaxy/web/base/pluginframework.py | 3 --- lib/galaxy/web/framework/base.py | 7 ------- .../web/framework/middleware/profile.py | 16 +++------------ lib/galaxy/web/security/__init__.py | 3 --- lib/galaxy/webapps/galaxy/api/histories.py | 4 ---- lib/galaxy/webapps/reports/buildapp.py | 3 --- .../webapps/reports/controllers/home.py | 2 -- .../webapps/reports/controllers/query.py | 2 -- lib/galaxy/webapps/tool_shed/buildapp.py | 3 --- 23 files changed, 27 insertions(+), 85 deletions(-) diff --git a/lib/galaxy/datatypes/display_applications/util.py b/lib/galaxy/datatypes/display_applications/util.py index 619e5ffb817..2aec95440aa 100644 --- a/lib/galaxy/datatypes/display_applications/util.py +++ b/lib/galaxy/datatypes/display_applications/util.py @@ -1,5 +1,3 @@ -import pkg_resources -pkg_resources.require( "pycrypto" ) from Crypto.Cipher import Blowfish diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 24b8c039891..f3501efde7d 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -2,9 +2,6 @@ Interval datatypes """ -import pkg_resources -pkg_resources.require( "bx-python" ) - import os import sys import logging @@ -1249,8 +1246,6 @@ class Wiggle( Tabular, _RemoteCallMixin ): Assumes we have a numpy file. """ # Maybe if we import here people will still be able to use Galaxy when numpy kills it - pkg_resources.require("numpy>=1.2.1") - import numpy range = end - start diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 30a362060db..447b566530a 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -2,8 +2,6 @@ Tabular datatype """ -import pkg_resources -pkg_resources.require( "bx-python" ) import gzip import logging import os diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 0b117acfc14..430b7393731 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -3,8 +3,6 @@ Provides utilities for working with GFF files. """ import copy -import pkg_resources -pkg_resources.require( "bx-python" ) from bx.intervals.io import GenomicInterval, MissingFieldError, NiceReaderWrapper, ParseError, GenomicIntervalReader from bx.tabular.io import Header, Comment from galaxy.util.odict import odict diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 9573644a71b..ff02cfb6257 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -1,6 +1,7 @@ """ Job control via the DRMAA API. """ +from __future__ import absolute_import import json import logging @@ -9,6 +10,16 @@ import string import subprocess import time +try: + import drmaa + DRMAA_IMPORT_MESSAGE = None +except (ImportError, RuntimeError) as exc: + drmaa = None + DRMAA_IMPORT_MESSAGE = ('The Python drmaa package is required to use this ' + 'feature, please install it or correct the ' + 'following error:\n%s: %s' % + (exc.__class__.__name__, str(exc))) + from galaxy import model from galaxy.jobs import JobDestination from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE @@ -19,8 +30,6 @@ log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] -drmaa = None - DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'workingDirectory', 'jobName', 'email', 'project' ] @@ -33,8 +42,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): def __init__( self, app, nworkers, **kwargs ): """Start the job runner""" - - global drmaa + assert drmaa is not None, DRMAA_IMPORT_MESSAGE runner_param_specs = dict( drmaa_library_path=dict( map=str, default=os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ), @@ -54,10 +62,6 @@ class DRMAAJobRunner( AsynchronousJobRunner ): log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path ) os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path - # We foolishly named this file the same as the name exported by the drmaa - # library... 'import drmaa' imports itself. - drmaa = __import__( "drmaa" ) - # Subclasses may need access to state constants self.drmaa_job_states = drmaa.JobState diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index 3147f274272..e7adfebb6f2 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -1,10 +1,19 @@ +from __future__ import absolute_import + import os import logging import time import traceback from datetime import timedelta -import pbs +try: + import pbs + PBS_IMPORT_MESSAGE = None +except ImportError as exc: + pbs = None + PBS_IMPORT_MESSAGE = ('The Python pbs-python package is required to use ' + 'this feature, please install it or correct the ' + 'following error:\nImportError %s' % str(exc)) from galaxy import model from galaxy import util @@ -82,8 +91,7 @@ class PBSJobRunner( AsynchronousJobRunner ): def __init__( self, app, nworkers ): """Start the job runner """ # Check if PBS was importable, fail if not - if pbs is None: - raise Exception( "PBSJobRunner requires pbs_python which was not found" ) + assert pbs is not None, PBS_IMPORT_MESSAGE if app.config.pbs_application_server and app.config.outputs_to_working_directory: raise Exception( "pbs_application_server (file staging) and outputs_to_working_directory options are mutually exclusive" ) diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 9124995845c..c12ae7a513c 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -25,12 +25,7 @@ attribute change to a model object. # instead of the three separate classes. With no 'apparent' perfect scheme # I'm opting to just keep them separate. -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy - -pkg_resources.require("repoze.lru") # used by Routes -pkg_resources.require("Routes") import routes from galaxy import exceptions diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 2fed11bec06..32a85acad19 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,10 +1,7 @@ -import pkg_resources - import functools import os import urllib2 -pkg_resources.require('Beaker') from beaker.cache import CacheManager from beaker.util import parse_cache_config_options diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index e85d869aa63..c41f1a8c1af 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -6,8 +6,6 @@ created (or copied) by users over the course of an analysis. """ import operator -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import desc, asc from galaxy import model diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 03456f7dde5..96e32ef45bb 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -2,8 +2,6 @@ Manager and Serializer for Users. """ -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy from galaxy import model diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py index b40730f09bf..008c5d9d0c8 100644 --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -8,8 +8,7 @@ import os import string import sys import tempfile -import pkg_resources -pkg_resources.require( "bx-python" ) + import bx.align.maf import bx.intervals import bx.interval_index_file diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index 88cde7c8678..8e50f8e34d9 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -1,6 +1,3 @@ -import pkg_resources -pkg_resources.require( "Cheetah" ) - from Cheetah.Template import Template diff --git a/lib/galaxy/visualization/plugins/plugin.py b/lib/galaxy/visualization/plugins/plugin.py index 16d75c250a2..cd354dd87aa 100644 --- a/lib/galaxy/visualization/plugins/plugin.py +++ b/lib/galaxy/visualization/plugins/plugin.py @@ -6,9 +6,6 @@ from a query string and render a webpage based on those data. import os import copy -import pkg_resources -pkg_resources.require( 'MarkupSafe' ) -pkg_resources.require( 'Mako' ) import mako from galaxy.managers import api_keys diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index e13c0e1f1fb..f9be1af8978 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -5,9 +5,6 @@ import logging import operator import re -import pkg_resources -pkg_resources.require("SQLAlchemy >= 0.4") - from sqlalchemy import true from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError diff --git a/lib/galaxy/web/base/pluginframework.py b/lib/galaxy/web/base/pluginframework.py index c2f323fa45b..f52e5f08023 100644 --- a/lib/galaxy/web/base/pluginframework.py +++ b/lib/galaxy/web/base/pluginframework.py @@ -16,9 +16,6 @@ from galaxy import util from galaxy.util import odict from galaxy.util import bunch -import pkg_resources -pkg_resources.require( 'MarkupSafe' ) -pkg_resources.require( 'Mako' ) import mako.lookup import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 845dda45e6c..85db7e3df42 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -9,13 +9,6 @@ import socket import tarfile import types -import pkg_resources - -pkg_resources.require("Paste") -pkg_resources.require("repoze.lru") # used by Routes -pkg_resources.require("Routes") -pkg_resources.require("WebOb") - import routes import webob diff --git a/lib/galaxy/web/framework/middleware/profile.py b/lib/galaxy/web/framework/middleware/profile.py index bb0dd9da185..d4fa6bcfb43 100644 --- a/lib/galaxy/web/framework/middleware/profile.py +++ b/lib/galaxy/web/framework/middleware/profile.py @@ -5,21 +5,11 @@ information at the bottom of each page. import threading import cgi +import cProfile +import pstats + from paste import response -try: - # Included in Python 2.5 - import cProfile -except: - try: - # Included in lsprof package for Python 2.4 - import pkg_resources - pkg_resources.require( "lsprof" ) - import cProfile - except: - cProfile = None - -import pstats template = """