From 49cc443b81868e7a98ddce45b7c33d952cc3f495 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 12 Feb 2015 12:13:24 -0500 Subject: [PATCH] Make it possible to disable egg fetching or the use of eggs entirely and provide a requirements.txt. --- config/galaxy.ini.sample | 28 ++++++++++++++++ lib/galaxy/eggs/__init__.py | 66 +++++++++++++++++++++++++++++-------- requirements.txt | 66 +++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 requirements.txt diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 814ef564443..6610e84347d 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -125,6 +125,34 @@ paste.app_factory = galaxy.web.buildapp:app_factory # but prefixed with install_ are also available). #install_database_connection = sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE +# -- Galaxy framework dependencies (eggs) + +# Options in this section can also be provided as environment variables in the +# form `GALAXY_`, e.g. `GALAXY_ENABLE_EGGS=False`, etc. +# Environment variables override the settings in this config. + +# For more details, see the `requirements.txt` file at the root of the Galaxy +# source distribution. + +# Galaxy manages all of its dependent Python modules by downloading Python eggs +# from the Galaxy eggs distribution server at https://eggs.galaxyproject.org. +# You may choose to disable this and attempt to install dependencies by hand +# (e.g. with pip), but beware that this may cause problems due to version +# incompatibilities between Galaxy and its dependencies. Do not do this unless +# you know what you're doing. +#enable_eggs = True + +# You may choose to keep eggs enabled but disable fetching them automatically. +# (hint: To do a one-time fetch, you can disable here and then use +# `GALAXY_ENABLE_EGG_FETCH=True python ./scripts/fetch_eggs.py` from the root +# of the Galaxy source. +#enable_egg_fetch = True + +# If you'd like to prefer any dependencies found in your local environment +# before (fetching and) using Galaxy's egg of that dependency, you can enable +# the following option: +#try_dependencies_from_env = False + # -- Files and directories # Dataset files are stored in this directory. diff --git a/lib/galaxy/eggs/__init__.py b/lib/galaxy/eggs/__init__.py index bb86f0a0da8..ff83f673afc 100644 --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -61,6 +61,7 @@ class Egg( object ): self.distribution = None self.dir = None self.removed_location = None + self.enable_fetch = crate.galaxy_config.enable_egg_fetch if self.name is not None and self.version is not None: self.set_distribution() @@ -191,7 +192,10 @@ class Egg( object ): try: rval = [] # resolve this egg and its dependencies - dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) + if self.enable_fetch: + dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) + else: + dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, lambda x: None ) for dist in dists: # if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ): @@ -250,7 +254,10 @@ class Egg( object ): if egg: # Store the removed path so the fetch method can use it egg.removed_location = location - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch ) + if self.enable_fetch: + r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch ) + else: + r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, lambda x: None ) egg.removed_location = None else: r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env ) @@ -374,31 +381,48 @@ class Crate( object ): """ Try to resolve (e.g. fetch) all eggs in the crate. """ - if all: - eggs = self.all_eggs + if self.galaxy_config.enable_eggs and not self.galaxy_config.try_dependencies_from_env: + if all: + eggs = self.all_eggs + else: + eggs = self.config_eggs + eggs = filter( lambda x: x.name not in self.no_auto, eggs ) + missing = [] + for egg in eggs: + try: + egg.resolve() + except EggNotFetchable: + missing.append( egg ) + if missing: + raise EggNotFetchable( missing ) else: - eggs = self.config_eggs - eggs = filter( lambda x: x.name not in self.no_auto, eggs ) - missing = [] - for egg in eggs: - try: - egg.resolve() - except EggNotFetchable: - missing.append( egg ) - if missing: - raise EggNotFetchable( missing ) + log.info('Dependencies will attempt to be loaded from the environment') class GalaxyConfig( object ): always_conditional = ( 'pysam', 'ctypes', 'python_daemon' ) def __init__( self, config_file ): + self.enable_egg_fetch = False + self.enable_eggs = False + self.try_dependencies_from_env = False if config_file is None: self.config = None else: self.config = ConfigParser.ConfigParser() if self.config.read( config_file ) == []: raise Exception( "error: unable to read Galaxy config from %s" % config_file ) + self.enable_egg_fetch = True + self.enable_eggs = True + for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): + try: + setattr(self, opt, string_as_bool(self.config.get('app:main', opt))) + except: + pass # use default + for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): + var = 'GALAXY_' + opt.upper() + if var in os.environ: + setattr(self, opt, string_as_bool(os.environ[var])) def check_conditional( self, egg_name ): def check_pysam(): @@ -435,6 +459,13 @@ class GalaxyConfig( object ): return False + +def string_as_bool( string ): + if str( string ).lower() in ( 'true', 'yes', 'on' ): + return True + else: + return False + def get_env(): env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() ) for dist in pkg_resources.find_distributions( eggs_dir, False ): @@ -446,6 +477,13 @@ env = get_env() def require( req_str ): c = Crate( None ) req = pkg_resources.Requirement.parse( req_str ) + if c.galaxy_config.try_dependencies_from_env or not c.galaxy_config.enable_eggs: + try: + return pkg_resources.working_set.require( req_str ) + except Exception as exc: + if not c.galaxy_config.enable_eggs: + raise + log.info("%s not found in local environment, will try Galaxy egg: %s", (req_str, exc)) # TODO: This breaks egg version requirements. Not currently a problem, but # it could become one. try: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000000..f32d1ea22f2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,66 @@ +# +# Galaxy has traditionally managed all of its own dependencies (in the form of +# precompiled Python eggs) for a few reasons: +# +# 1. We can ensure that Galaxy works with the versions of the dependencies that +# we require. +# 2. We can precompile them for numerous platforms, meaning that developers or +# system administrators attempting to use Galaxy do not need to wait while +# dependencies compile/install, nor do they need to have compilers or +# various development libraries installed on their system. +# +# However, you may want to manage these dependencies on your own. If so, this +# file can be used with pip to install all of the bare minimum dependencies for +# starting a Galaxy server. Use this in conjunction with the +# `enable_eggs = False` galaxy.ini option or `GALAXY_ENABLE_EGGS=False` +# environment variable +# +# Note that except in the case of SQLAlchemy, all dependencies will be +# installed at their newest versions. Galaxy has not been tested with these and +# you may (likely?) run into problems! The Galaxy Team does not support using +# Galaxy with dependency versions other than those in eggs.ini, however, if you +# do find an incompatibility between Galaxy and the latest version, we would +# appreciate if you would investigate and provide a code fix for the issue. We +# are trying to move away from our strict version dependency policy and this +# would greatly help in our efforts. + +# not available in PyPI, must be installed by hand: +#SVGFig + +Paste +PasteDeploy +docutils +wchartype +pexpect +amqp +repoze.lru +Routes +WebOb +SQLAlchemy<0.8 +Cheetah +pycrypto +MarkupSafe +WebHelpers +anyjson +kombu +numpy +bx-python # bx must be installed after numpy! +pysam +Mako +PyYAML +mercurial +pytz +Babel +decorator +Tempita +sqlalchemy-migrate +Beaker +Whoosh +paramiko +ssh +Fabric +boto +simplejson +bioblend +Parsley +WebError