diff --git a/create_db.sh b/create_db.sh index 857dc98feaf..2f3243420ec 100755 --- a/create_db.sh +++ b/create_db.sh @@ -1,4 +1,10 @@ #!/bin/sh +if [ -d .venv ]; +then + printf "Activating virtualenv at %s/.venv\n" $(pwd) + . .venv/bin/activate +fi + cd `dirname $0` python ./scripts/create_db.py $@ diff --git a/dist-eggs.ini b/dist-eggs.ini deleted file mode 100644 index 92146d83bce..00000000000 --- a/dist-eggs.ini +++ /dev/null @@ -1,52 +0,0 @@ -; -; Config for building eggs for distribution (via a site such as -; eggs.galaxyproject.org). Probably only useful to members of the Galaxy Team -; building eggs for distribution. This file is used by -; scripts/dist-scramble.py -; -; More information: http://wiki.galaxyproject.org/Admin/Config/Eggs -; - -[hosts] -py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-i686-ucs2/bin/python2.6 -py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-i686-ucs4/bin/python2.6 -py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs2/bin/python2.6 -py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs4/bin/python2.6 -py2.7-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-i686-ucs2/bin/python2.7 -py2.7-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-i686-ucs4/bin/python2.7 -py2.7-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs2/bin/python2.7 -py2.7-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs4/bin/python2.7 -py2.6-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.6 -py2.7-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.7 -; -; these hosts are used to build eggs with no C extensions -py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs4/bin/python2.6 -py2.7 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs4/bin/python2.7 - -[groups] -py2.6-linux-i686 = py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 -py2.6-linux-x86_64 = py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 -py2.7-linux-i686 = py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 -py2.7-linux-x86_64 = py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 -py2.6-linux = py2.6-linux-i686 py2.6-linux-x86_64 -py2.7-linux = py2.7-linux-i686 py2.7-linux-x86_64 -linux-i686 = py2.6-linux-i686 py2.7-linux-i686 -linux-x86_64 = py2.6-linux-x86_64 py2.7-linux-x86_64 -linux = linux-i686 linux-x86_64 -py2.6-macosx = py2.6-macosx-10.6-intel-ucs2 -py2.7-macosx = py2.7-macosx-10.6-intel-ucs2 -macosx = py2.6-macosx py2.7-macosx -py2.6-all = py2.6-linux py2.6-macosx -py2.7-all = py2.7-linux py2.7-macosx - -; the 'all' key is used internally by the build system to specify which hosts -; to build on when no hosts are specified on the dist-eggs.py command line. -all = linux macosx - -; the 'noplatform' key, likewise, is for which build hosts should be used when -; building pure python (noplatform) eggs. -noplatform = py2.6 py2.7 - -; don't build these eggs on these platforms: -[ignore] -;ctypes = diff --git a/eggs.ini b/eggs.ini deleted file mode 100644 index 12a9f88f3e9..00000000000 --- a/eggs.ini +++ /dev/null @@ -1,96 +0,0 @@ -; -; Galaxy egg version configuration -; -; This file is version controlled and should not be edited by hand! -; For more information, see: -; https://wiki.galaxyproject.org/Admin/Config/Eggs -; - -[general] -repository = http://eggs.galaxyproject.org -; these eggs must be scrambled for your local environment -no_auto = pbs_python - -[eggs:platform] -bx_python = 0.7.2 -Cheetah = 2.2.2 -MarkupSafe = 0.12 -mercurial = 3.2.4 -MySQL_python = 1.2.3c1 -PyRods = 3.2.4 -numpy = 1.6.0 -pbs_python = 4.3.5 -psycopg2 = 2.5.1 -pycrypto = 2.5 -pysam = 0.4.2 -pysqlite = 2.5.6 -python_lzo = 1.08_2.03_static -PyYAML = 3.10 -SQLAlchemy = 1.0.8 -; msgpack_python = 0.2.4 - -[eggs:noplatform] -amqp = 1.4.6 -anyjson = 0.3.3 -Beaker = 1.4 -bioblend = 0.5.2 -boto = 2.27.0 -decorator = 3.1.2 -docutils = 0.7 -drmaa = 0.7.6 -Fabric = 1.7.0 -importlib = 1.0.3 -kombu = 3.0.24 -lrucache = 0.2 -Mako = 0.4.1 -mock = 1.0.1 -nose = 1.3.6 -NoseHTML = 0.4.1 -NoseTestDiff = 0.1 -ordereddict = 1.1 -paramiko = 1.11.1 -Parsley = 1.1 -Paste = 1.7.5.1 -PasteDeploy = 1.5.0 -pexpect = 2.4 -poster = 0.8.1 -python_openid = 2.2.5 -python_daemon = 1.5.5 -pytz = 2013.9 -repoze.lru = 0.6 -requests = 2.2.1 -Routes = 2.1 -simplejson = 2.1.1 -six = 1.9.0 -sqlalchemy_migrate = 0.9.6 -sqlparse = 0.1.14 -ssh = 1.7.14 -SVGFig = 1.1.6 -Tempita = 0.5.1 -twill = 0.9 -WebError = 0.8a -WebHelpers = 1.3 -WebOb = 0.8.5 -wsgiref = 0.1.2 -Babel = 1.3 -wchartype = 0.1 -Whoosh = 2.4.1 -; fluent_logger = 0.3.3 -raven = 3.1.8 - -; extra version information -[tags] -psycopg2 = _9.2.4_static -pysqlite = _3.6.17_static -MySQL_python = _5.1.41_static -pysam = _kanwei_b10f6e722e9a - -; dependency source urls, necessary for scrambling. for an explanation, see -; the wiki page above -[source] -MySQL_python = mysql-5.1.41 -psycopg2 = postgresql-9.2.4 -pysqlite = sqlite-amalgamation-3_6_17 - -[dependencies] -bx_python = numpy diff --git a/lib/galaxy/__init__.py b/lib/galaxy/__init__.py index 61343e2c0ac..07a3f6c52a0 100644 --- a/lib/galaxy/__init__.py +++ b/lib/galaxy/__init__.py @@ -2,82 +2,9 @@ Galaxy root package -- this is a namespace package. """ -__import__( "pkg_resources" ).declare_namespace( __name__ ) +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) -import sys -import platform - -import pkg_resources - - -# patch get_platform() for better ABI recognition -def _get_build_platform(): - plat = pkg_resources._get_build_platform() - if sys.platform == 'darwin': - # Assert OS X version is new enough - current_minor_ver = int( platform.mac_ver()[0].split( '.' )[1] ) - assert current_minor_ver >= 6, 'Galaxy is not compatible with Mac OS X < 10.6 (your version is %s)' % platform.mac_ver()[0] - # Python build target may be even older, but this will prevent it from - # fetching our 10.6+ eggs, so fix if necessary (newer versions will be - # fine) - plat_split = plat.split( '-' ) - build_minor_ver = int( plat_split[1].split( '.' )[1] ) - if build_minor_ver < 6: - plat_split[1] = '10.6' - # Our intel (dual arch) eggs will work fine on single-arch builds - if plat_split[-1] in ( 'i386', 'x86_64' ): - plat_split[-1] = 'intel' - # Ditto universal (if you're not on PPC) - if plat_split[-1] == 'universal' and platform.processor() != 'powerpc': - plat_split[-1] = 'intel' - plat = '-'.join( plat_split ) - elif sys.platform == "linux2" and sys.maxint < 2 ** 31 and plat.endswith( '-x86_64' ): - # 32 bit Python on 64 bit Linux - plat = plat.replace( '-x86_64', '-i686' ) - if not (plat.endswith('-ucs2') or plat.endswith('-ucs4')): - if sys.maxunicode > 2 ** 16: - plat += '-ucs4' - else: - plat += '-ucs2' - return plat -try: - assert pkg_resources._get_build_platform -except: - pkg_resources._get_build_platform = pkg_resources.get_build_platform - pkg_resources.get_build_platform = _get_build_platform - pkg_resources.get_platform = _get_build_platform - - -# patch to insert eggs at the beginning of sys.path instead of at the end -def _insert_on(self, path, loc=None): - """Insert self.location in path before its nearest parent directory""" - - loc = loc or self.location - if not loc: - return - - nloc = pkg_resources._normalize_cached(loc) - npath = [(p and pkg_resources._normalize_cached(p) or p) for p in path] - - if path is sys.path: - self.check_version_conflict() - path.insert(0, loc) - - # remove dups - while 1: - try: - np = npath.index(nloc, 1) - except ValueError: - break - else: - del npath[np], path[np] - - return -try: - assert pkg_resources.Distribution._insert_on -except: - pkg_resources.Distribution._insert_on = pkg_resources.Distribution.insert_on - pkg_resources.Distribution.insert_on = _insert_on # compat: BadZipFile introduced in Python 2.7 import zipfile diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 2a9dc3f73f5..2955d7fa0ad 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -14,7 +14,6 @@ import string import sys import tempfile from datetime import timedelta -from galaxy import eggs from galaxy.exceptions import ConfigurationError from galaxy.util import listify from galaxy.util import string_as_bool @@ -729,7 +728,6 @@ def configure_logging( config ): root.addHandler( handler ) # If sentry is configured, also log to it if config.sentry_dsn: - eggs.require( "raven" ) from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 6664cebe576..5bbad2e4dd0 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -8,11 +8,10 @@ import shutil import struct import subprocess import tempfile -import warnings import zipfile -from galaxy import eggs -eggs.require( "bx-python" ) +import pysam + from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter @@ -20,11 +19,6 @@ from galaxy.datatypes import metadata from galaxy.util import nice_size, sqlite from . import data, dataproviders -with warnings.catch_warnings(): - warnings.simplefilter( "ignore" ) - eggs.require( "pysam" ) - from pysam import csamtools - log = logging.getLogger(__name__) @@ -340,7 +334,7 @@ class Bam( Binary ): os.unlink( stderr_name ) # Now use pysam with BAI index to determine additional metadata try: - bam_file = csamtools.Samfile( filename=dataset.file_name, mode='rb', index_filename=index_file.file_name ) + bam_file = pysam.AlignmentFile( filename=dataset.file_name, mode='rb', index_filename=index_file.file_name ) dataset.metadata.reference_names = list( bam_file.references ) dataset.metadata.reference_lengths = list( bam_file.lengths ) dataset.metadata.bam_header = bam_file.header @@ -348,6 +342,8 @@ class Bam( Binary ): dataset.metadata.sort_order = dataset.metadata.bam_header.get( 'HD', {} ).get( 'SO', None ) dataset.metadata.bam_version = dataset.metadata.bam_header.get( 'HD', {} ).get( 'VN', None ) except: + # Per Dan, don't log here because doing so will cause datasets that + # fail metadata to end in the error state pass def sniff( self, filename ): diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py index f66b1d8da27..5180863f127 100644 --- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py @@ -4,10 +4,6 @@ from __future__ import division import sys -from galaxy import eggs - -eggs.require('numpy') # noqa -eggs.require('bx-python') # noqa from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict BLOCK_SIZE = 100 diff --git a/lib/galaxy/datatypes/converters/bgzip.py b/lib/galaxy/datatypes/converters/bgzip.py index b3be76c2449..5d2547019f8 100644 --- a/lib/galaxy/datatypes/converters/bgzip.py +++ b/lib/galaxy/datatypes/converters/bgzip.py @@ -9,9 +9,7 @@ import optparse import subprocess import tempfile -from galaxy import eggs -eggs.require( "pysam" ) -import ctabix +from pysam import ctabix def main(): diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py index 051623cdbd3..eaca2e6240d 100644 --- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py @@ -11,8 +11,6 @@ from __future__ import division import fileinput import sys -from galaxy import eggs -eggs.require( "bx-python" ) from bx.interval_index_file import Indexes from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GenomicInterval, GFFReaderWrapper diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py index a987cbab5ec..6942449d3b2 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py @@ -2,8 +2,6 @@ # Dan Blankenberg import sys -from galaxy import eggs -eggs.require( "bx-python" ) import bx.intervals.io assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py index 42d7af5800b..87ef467807f 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py @@ -2,8 +2,6 @@ # Dan Blankenberg import sys -from galaxy import eggs -eggs.require( "bx-python" ) import bx.intervals.io assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index bba6e236397..1d367ccb5fb 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -11,8 +11,6 @@ import tempfile from bisect import bisect from os import environ -from galaxy import eggs -eggs.require( "bx-python" ) from bx.cookbook import doc_optparse from bx.intervals import io diff --git a/lib/galaxy/datatypes/converters/interval_to_fli.py b/lib/galaxy/datatypes/converters/interval_to_fli.py index 4d3d321c610..94d94601583 100644 --- a/lib/galaxy/datatypes/converters/interval_to_fli.py +++ b/lib/galaxy/datatypes/converters/interval_to_fli.py @@ -15,8 +15,6 @@ and symbols are sorted in lexigraphical order. ''' import optparse -from galaxy import eggs -eggs.require( "bx-python" ) from bx.tabular.io import Comment from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFReaderWrapper, read_unordered_gtf diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py index f818060e605..901816d912c 100644 --- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py @@ -12,8 +12,6 @@ usage: %prog in_file out_file from __future__ import division import optparse -from galaxy import eggs -eggs.require( "bx-python" ) from bx.interval_index_file import Indexes diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py index abc7ce90aba..dc2fa1f938e 100644 --- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py @@ -10,9 +10,7 @@ import optparse import os import sys -from galaxy import eggs -eggs.require( "pysam" ) -import ctabix +from pysam import ctabix def main(): diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index 9964b079fc6..068cf8584bc 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -2,8 +2,6 @@ # Dan Blankenberg import sys -from galaxy import eggs -eggs.require( "bx-python" ) import bx.align.maf from galaxy.tools.util import maf_utilities diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index dad9e07cb50..812fdeac25f 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -2,8 +2,6 @@ # Dan Blankenberg import sys -from galaxy import eggs -eggs.require( "bx-python" ) import bx.align.maf from galaxy.tools.util import maf_utilities diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py index 4142cff235e..9c879dc05bb 100644 --- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py @@ -9,8 +9,6 @@ usage: %prog in_file out_file from __future__ import division import optparse -from galaxy import eggs -eggs.require( "bx-python" ) from bx.interval_index_file import Indexes diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py index 34281d59c73..076cd6e2f0b 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py @@ -7,8 +7,6 @@ from __future__ import division import optparse -from galaxy import eggs -eggs.require( "bx-python" ) from bx.interval_index_file import Indexes import galaxy_utils.sequence.vcf diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py index 8464b17e88a..eebf5cc5278 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py +++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py @@ -9,9 +9,7 @@ usage: %prog in_file out_file """ import optparse -from galaxy import eggs -eggs.require( "pysam" ) -import ctabix +from pysam import ctabix def main(): diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py index 6332c4ead8a..f5201f6c290 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py @@ -4,9 +4,6 @@ from __future__ import division import sys -from galaxy import eggs -eggs.require('numpy') # noqa -eggs.require('bx-python') # noqa from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict from bx.arrays.wiggle import WiggleReader diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py index 7e408f3a2d9..a409745aff8 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py @@ -8,8 +8,6 @@ and fixedStep wiggle lines. """ import sys -from galaxy import eggs -eggs.require( "bx-python" ) import bx.wiggle from galaxy.tools.exception_handling import UCSCOutWrapper, UCSCLimitException diff --git a/lib/galaxy/datatypes/coverage.py b/lib/galaxy/datatypes/coverage.py index 840756fbaf1..5a3c7d60cbc 100644 --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -6,7 +6,6 @@ Coverage datatypes import logging import math -from galaxy import eggs from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.tabular import Tabular @@ -28,7 +27,6 @@ class LastzCoverage( Tabular ): Assumes we have a numpy file. """ # Maybe if we import here people will still be able to use Galaxy when numpy kills it - eggs.require("numpy>=1.2.1") import numpy range = end - start diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 07a666b6a1c..e1db278e001 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -17,8 +17,6 @@ from galaxy.util.sanitize_html import sanitize_html import dataproviders -from galaxy import eggs -eggs.require( "Paste" ) import paste XSS_VULNERABLE_MIME_TYPES = [ diff --git a/lib/galaxy/datatypes/dataproviders/dataset.py b/lib/galaxy/datatypes/dataproviders/dataset.py index b692f271ef8..6547854567d 100644 --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -14,8 +14,6 @@ import external from galaxy.util import sqlite import sys -from galaxy import eggs -eggs.require( 'bx-python' ) from bx import seq as bx_seq from bx import wiggle as bx_wig from bx import bbi as bx_bbi diff --git a/lib/galaxy/datatypes/display_applications/util.py b/lib/galaxy/datatypes/display_applications/util.py index 68ab51abe5f..2aec95440aa 100644 --- a/lib/galaxy/datatypes/display_applications/util.py +++ b/lib/galaxy/datatypes/display_applications/util.py @@ -1,5 +1,3 @@ -from galaxy import eggs -eggs.require( "pycrypto" ) from Crypto.Cipher import Blowfish diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 998ee9318a6..07fc6c75550 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -8,9 +8,6 @@ import sys import tempfile import urllib -from galaxy import eggs -eggs.require( "bx-python" ) -eggs.require( "numpy" ) import numpy from bx.intervals.io import GenomicIntervalReader, ParseError diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 2b92652036b..7c4a11a9cfa 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -14,8 +14,6 @@ import weakref from os.path import abspath -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy.orm import object_session import galaxy.model diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 71dea891f40..720c10c513b 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -10,7 +10,7 @@ import re import string from cgi import escape -from galaxy import eggs, util +from galaxy import util from galaxy.datatypes import metadata from galaxy.datatypes.checkers import is_gzip from galaxy.datatypes.metadata import MetadataElement @@ -19,11 +19,7 @@ from galaxy.datatypes.util.image_util import check_image_type from galaxy.util import nice_size from . import data -try: - eggs.require( "bx-python" ) - import bx.align.maf -except: - pass +import bx.align.maf log = logging.getLogger(__name__) diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 8146211f252..68e35adf481 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -3,8 +3,6 @@ Provides utilities for working with GFF files. """ import copy -from galaxy import eggs -eggs.require( "bx-python" ) from bx.intervals.io import GenomicInterval, MissingFieldError, NiceReaderWrapper, ParseError, GenomicIntervalReader from bx.tabular.io import Header, Comment diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py new file mode 100644 index 00000000000..27dda0ed5d7 --- /dev/null +++ b/lib/galaxy/dependencies/__init__.py @@ -0,0 +1,78 @@ +""" +Determine what optional dependencies are needed. +""" +import pkg_resources + +from os.path import dirname, join +from xml.etree import ElementTree + +from galaxy.util import asbool +from galaxy.util.properties import load_app_properties + + +class ConditionalDependencies( object ): + def __init__( self, config_file ): + self.config_file = config_file + self.config = None + self.job_runners = [] + self.conditional_reqs = [] + self.parse_configs() + self.get_conditional_requirements() + + def parse_configs( self ): + self.config = load_app_properties( ini_file=self.config_file ) + job_conf_xml = self.config.get( + "job_config_file", + join( dirname( self.config_file ), 'job_conf.xml' ) ) + try: + for plugin in ElementTree.parse( job_conf_xml ).find( 'plugins' ): + self.job_runners.append( plugin.attrib['load'] ) + except (OSError, IOError): + pass + + def get_conditional_requirements( self ): + crfile = join( dirname( __file__ ), 'conditional-requirements.txt' ) + for req in pkg_resources.parse_requirements( open( crfile ).readlines() ): + self.conditional_reqs.append( req ) + + def check( self, name ): + try: + name = name.replace('-', '_').replace('.', '_') + return getattr( self, 'check_' + name )() + except: + return False + + def check_psycopg2( self ): + return self.config["database_connection"].startswith( "postgres" ) + + def check_mysql_python( self ): + return self.config["database_connection"].startswith( "mysql" ) + + def check_drmaa( self ): + return ("galaxy.jobs.runners.drmaa:DRMAAJobRunner" in self.job_runners or + "galaxy.jobs.runners.slurm:SlurmJobRunner" in self.job_runners) + + def check_pbs_python( self ): + return "galaxy.jobs.runners.pbs:PBSJobRunner" in self.job_runners + + def check_python_openid( self ): + return asbool( self.config["enable_openid"] ) + + def check_fluent_logger( self ): + return asbool( self.config["fluent_log"] ) + + def check_raven( self ): + return asbool( self.config["sentry_dsn"] ) + + def check_weberror( self ): + return ( asbool( self.config["debug"] ) and + asbool( self.config["use_interactive"] ) ) + + +def optional( config_file ): + rval = [] + conditional = ConditionalDependencies( config_file ) + for opt in conditional.conditional_reqs: + if conditional.check( opt.key ): + rval.append( str( opt ) ) + return rval diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt new file mode 100644 index 00000000000..e72f8a4a3f9 --- /dev/null +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -0,0 +1,10 @@ +# These dependencies are only required when certain config options are set +psycopg2==2.6.1 +WebError==0.11 +python-openid +MySQL-python +fluent-logger +raven +pbs_python +drmaa +# PyRods not in PyPI diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt new file mode 100644 index 00000000000..10fa7da2a1c --- /dev/null +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -0,0 +1,3 @@ +nose +NoseHTML +twill==0.9.1 diff --git a/lib/galaxy/dependencies/requirements.txt b/lib/galaxy/dependencies/requirements.txt new file mode 100644 index 00000000000..677ae2d4980 --- /dev/null +++ b/lib/galaxy/dependencies/requirements.txt @@ -0,0 +1,64 @@ +# packages with C extensions +Cheetah==2.4.4 +bx-python==0.7.3 +MarkupSafe==0.23 +PyYAML==3.11 +SQLAlchemy==1.0.8 +mercurial==3.5 +numpy==1.9.2 +pycrypto==2.6.1 +python_lzo==1.8 + +# extra functionality from Dannon +pysam==0.8.3+gx1 + +# pure Python packages +Paste==1.7.5.1 +PasteDeploy==1.5.2 +docutils==0.12 +wchartype==0.1 +repoze.lru==0.6 +Routes==2.2 +WebOb==1.4.1 +WebHelpers==1.3 +Mako==1.0.2 +pytz==2015.4 +Babel==2.0 +Beaker==1.7.0 + +# BioBlend and dependencies +bioblend==0.6.1 +boto==2.38.0 +requests==2.8.1 + +# WebError and dependences +WebError==0.11 +Pygments==2.0.2 + +# kombu and dependencies +kombu==3.0.26 +amqp==1.4.6 +anyjson==0.3.3 + +# sqlalchemy-migrate and dependencies +sqlalchemy-migrate==0.10.0 +decorator==4.0.2 +Tempita==0.5.3dev +sqlparse==0.1.16 +pbr==1.8.0 +six==1.9.0 # six is also a Pulsar client dep +Parsley==1.3 +nose==1.3.7 +SVGFig==1.1.6 + +# Fabric and dependencies +Fabric==1.10.2 +paramiko==1.15.2 +ecdsa==0.13 +# pycrypto, but it is a direct Galaxy dependency as well + +# Can't use Whoosh latest due to a bug but need to backport a bugfix from a +# newer release. +# +# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff +Whoosh==2.4.1+gx1 diff --git a/lib/galaxy/eggs/__init__.py b/lib/galaxy/eggs/__init__.py index f778bfe4e46..95e6a1707da 100644 --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -1,538 +1,8 @@ """ -Manage Galaxy eggs +For backwards compatibility """ -import ConfigParser -import glob -import HTMLParser -import os import pkg_resources -import shutil -import sys -import urllib -import urllib2 -import zipfile -import zipimport - -import logging -log = logging.getLogger( __name__ ) -log.addHandler( logging.NullHandler() ) -galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir ) ) -eggs_dir = os.environ.get( 'GALAXY_EGGS_PATH', os.path.join( galaxy_dir, 'eggs' ) ) -py = 'py%s' % sys.version[:3] - - -class EggNotFetchable( Exception ): - def __init__( self, eggs ): - if type( eggs ) in ( list, tuple ): - self.eggs = eggs - else: - self.eggs = [ eggs ] - - def __str__( self ): - return ' '.join( self.eggs ) - - -# need the options to remain case sensitive -class CaseSensitiveConfigParser( ConfigParser.SafeConfigParser ): - def optionxform( self, optionstr ): - return optionstr - - -# so we can actually detect failures -class URLRetriever( urllib.FancyURLopener ): - def http_error_default( *args ): - urllib.URLopener.http_error_default( *args ) - - -class Egg( object ): - """ - Contains information about locating and downloading eggs. - """ - def __init__( self, name=None, version=None, tag=None, url=None, platform=None, crate=None ): - self.name = name - self.version = version - self.tag = tag - self.url = url - self.platform = platform - self.crate = crate - self.distribution = None - self.dir = None - self.removed_location = None - self.enable_fetch = crate.enable_egg_fetch - if self.name is not None and self.version is not None: - self.set_distribution() - - def set_dir( self ): - global eggs_dir - self.dir = eggs_dir - - def set_distribution( self ): - """ - Stores a pkg_resources Distribution object for reference later - """ - if self.dir is None: - self.set_dir() - tag = self.tag or '' - self.distribution = pkg_resources.Distribution.from_filename( - os.path.join( self.dir, '-'.join( ( self.name, self.version + tag, self.platform ) ) + '.egg' ) ) - - @property - def path( self ): - """ - Return the path of the egg, if it exists, or None - """ - if env[self.distribution.project_name]: - return env[self.distribution.project_name][0].location - return None - - def fetch( self, requirement ): - """ - fetch() serves as the install method to pkg_resources.working_set.resolve() - """ - def find_alternative(): - """ - Some platforms (e.g. Solaris) support eggs compiled on older platforms - """ - class LinkParser( HTMLParser.HTMLParser ): - """ - Finds links in what should be an Apache-style directory index - """ - def __init__( self ): - HTMLParser.HTMLParser.__init__( self ) - self.links = [] - - def handle_starttag( self, tag, attrs ): - if tag == 'a' and 'href' in dict( attrs ): - self.links.append( dict( attrs )['href'] ) - - parser = LinkParser() - try: - parser.feed( urllib2.urlopen( self.url + '/' ).read() ) - except urllib2.HTTPError, e: - if e.code == 404: - return None - parser.close() - for link in parser.links: - file = urllib.unquote( link ).rsplit( '/', 1 )[-1] - tmp_dist = pkg_resources.Distribution.from_filename( file ) - if tmp_dist.platform is not None and \ - self.distribution.project_name == tmp_dist.project_name and \ - self.distribution.version == tmp_dist.version and \ - self.distribution.py_version == tmp_dist.py_version and \ - pkg_resources.compatible_platforms( tmp_dist.platform, pkg_resources.get_platform() ): - return file - return None - - def _fetch(): - if self.url is None: - return False - if not os.path.exists( self.dir ): - os.makedirs( self.dir ) - alternative = None - try: - url = self.url + '/' + self.distribution.egg_name() + '.egg' - URLRetriever().retrieve( url, self.distribution.location ) - log.debug( "Fetched %s" % url ) - except IOError, e: - if e[1] == 404 and self.distribution.platform != py: - alternative = find_alternative() - if alternative is None: - return False - else: - return False - if alternative is not None: - try: - url = '/'.join( ( self.url, alternative ) ) - URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) ) - log.debug( "Fetched %s" % url ) - except IOError, e: - return False - self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0] - self.set_distribution() - self.unpack_if_needed() - self.remove_doppelgangers() - return True - # If being called from a version conflict, that code has removed a - # directory from sys.path. That directory could be Python's main lib - # directory (e.g. in the case of wsgiref under 2.6). Temporarily put - # it back during the execution of this code. - if self.removed_location: - sys.path.append( self.removed_location ) - try: - assert _fetch() - rval = self.distribution - except: - rval = None - if self.removed_location: - sys.path.remove( self.removed_location ) - if rval is not None: - global env - env = get_env() # reset the global Environment object now that we've obtained a new egg - return rval - - def unpack_if_needed( self ): - meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.distribution.location ) ) - if meta.has_metadata( 'not-zip-safe' ): - unpack_zipfile( self.distribution.location, self.distribution.location + "-tmp" ) - os.remove( self.distribution.location ) - os.rename( self.distribution.location + "-tmp", self.distribution.location ) - - def remove_doppelgangers( self ): - doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % ( self.name, self.platform ) ) ) - if self.distribution.location in doppelgangers: - doppelgangers.remove( self.distribution.location ) - for doppelganger in doppelgangers: - remove_file_or_path( doppelganger ) - log.debug( "Removed conflicting egg: %s" % doppelganger ) - - def resolve( self ): - try: - rval = [] - # resolve this egg and its dependencies - if self.enable_fetch: - dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch ) - else: - dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, lambda x: None ) - for dist in dists: - # if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them - if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ): - # TODO: like eggs.require(), this breaks requirement versioning - subdists = self.version_conflict( dist, dist.as_requirement() ) - if type( subdists ) == list: - rval.extend( subdists ) - else: - rval.append( subdists ) - else: - rval.append( dist ) - return rval - except pkg_resources.DistributionNotFound, e: - # If this statement is true, it means we do have the requested egg, - # just not one (or more) of its deps. - if e.args[0].project_name != self.distribution.project_name: - log.warning( "Warning: %s (a dependent egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) ) - return ( self.distribution, ) - else: - raise EggNotFetchable( self ) - except pkg_resources.VersionConflict, e: - if e.args[1].key != e.args[0].key: - log.error( "One of Galaxy's managed eggs depends on something which is missing, this is almost certainly a bug in the egg distribution." ) - log.error( 'Dependency "%s" requires "%s"' % ( e.args[0].project_name, str( e.args[1] ) ) ) - raise - # there's a conflicting egg on the path, remove it - return self.version_conflict( e.args[0], e.args[1] ) - - def version_conflict( self, conflict_dist, conflict_req ): - # since this conflict may be for a dependent egg, find the correct egg from the crate - if conflict_dist.project_name == self.distribution.project_name: - egg = self - dist = egg.distribution - elif conflict_dist.project_name in self.crate.all_names: - egg = self.crate[conflict_dist.project_name] - dist = egg.distribution - else: - # should not happen, but just in case - egg = None - dist = conflict_dist - # use the canonical path to locate and remove the conflict from the working set - location = os.path.realpath( conflict_dist.location ) - for entry in pkg_resources.working_set.entries: - if os.path.realpath( entry ) == location: - pkg_resources.working_set.entries.remove( entry ) - break - else: - location = entry = None - del pkg_resources.working_set.by_key[conflict_dist.key] - # remove the conflict from sys.path - if entry is not None: - pkg_resources.working_set.entry_keys[entry] = [] - if entry in sys.path: - sys.path.remove(entry) - # if the conflict is a dependent egg, fetch that specific egg - if egg: - # Store the removed path so the fetch method can use it - egg.removed_location = location - if self.enable_fetch: - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch ) - else: - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, lambda x: None ) - egg.removed_location = None - else: - r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env ) - # re-add the path if it's a non-egg dir, in case more deps live there - if location is not None and not location.endswith( '.egg' ): - pkg_resources.working_set.entries.append( location ) - sys.path.append( location ) - return r - - def require( self ): - try: - dists = self.resolve() - for dist in dists: - if dist.location not in pkg_resources.working_set.entries: - pkg_resources.working_set.add( dist ) - return dists - except: - raise - - -class Crate( object ): - """ - Reads the eggs.ini file for use with checking and fetching. - """ - config_file = os.path.join( galaxy_dir, 'eggs.ini' ) - - def __init__( self, galaxy_config_file=None, platform=None ): - self.eggs = {} - self.config = CaseSensitiveConfigParser() - self.repo = None - self.no_auto = [] - self.platform = platform - self.py_platform = None - if platform is not None: - self.py_platform = platform.split( '-' )[0] - self.enable_egg_fetch = string_as_bool(os.environ.get('GALAXY_ENABLE_EGG_FETCH', True)) - self.enable_eggs = string_as_bool(os.environ.get('GALAXY_ENABLE_EGGS', True)) - self.try_dependencies_from_env = string_as_bool(os.environ.get('GALAXY_TRY_DEPENDENCIES_FROM_ENV', False)) - self.galaxy_config = GalaxyConfig( galaxy_config_file ) - self.parse() - - def parse( self ): - self.config.read( Crate.config_file ) - self.repo = self.config.get( 'general', 'repository' ) - self.no_auto = self.config.get( 'general', 'no_auto' ).split() - self.parse_egg_section( self.config.items( 'eggs:platform' ), self.config.items( 'tags' ), True ) - self.parse_egg_section( self.config.items( 'eggs:noplatform' ), self.config.items( 'tags' ) ) - - def parse_egg_section( self, eggs, tags, full_platform=False, egg_class=Egg ): - for name, version in eggs: - tag = dict( tags ).get( name, '' ) - url = '/'.join( ( self.repo, name ) ) - if full_platform: - platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) ) - else: - platform = self.py_platform or py - egg = egg_class( name=name, version=version, tag=tag, url=url, platform=platform, crate=self ) - self.eggs[name] = egg - - @property - def config_missing( self ): - """ - Return true if any eggs are missing, conditional on options set in the - Galaxy config file. - """ - for egg in self.config_eggs: - if not egg.path: - return True - return False - - @property - def all_missing( self ): - """ - Return true if any eggs in the eggs config file are missing. - """ - for egg in self.all_eggs: - if not os.path.exists( egg.distribution.location ): - return True - return False - - @property - def config_names( self ): - """ - Return a list of names of all eggs in the crate that are needed based - on the options set in the Galaxy config file. - """ - return [ egg.name for egg in self.config_eggs ] - - @property - def all_names( self ): - """ - Return a list of names of all eggs in the crate. - """ - return [ egg.name for egg in self.all_eggs ] - - @property - def config_eggs( self ): - """ - Return a list of all eggs in the crate that are needed based on the - options set in the Galaxy config file. - """ - return [ egg for egg in self.eggs.values() if self.galaxy_config.check_conditional( egg.name ) ] - - @property - def all_eggs( self ): - """ - Return a list of all eggs in the crate. - """ - rval = [] - for egg in self.eggs.values(): - if egg.name in self.galaxy_config.always_conditional and not self.galaxy_config.check_conditional( egg.name ): - continue - rval.append( egg ) - return rval - - def __getitem__( self, name ): - """ - Return a specific egg. - """ - name = name.replace( '-', '_' ) - return self.eggs[name] - - def resolve( self, all=False ): - """ - Try to resolve (e.g. fetch) all eggs in the crate. - """ - if self.enable_eggs and not self.try_dependencies_from_env: - if all: - eggs = self.all_eggs - else: - eggs = self.config_eggs - eggs = filter( lambda x: x.name not in self.no_auto, eggs ) - missing = [] - for egg in eggs: - try: - egg.resolve() - except EggNotFetchable: - missing.append( egg ) - if missing: - raise EggNotFetchable( missing ) - else: - log.info('Dependencies will attempt to be loaded from the environment') - - -class GalaxyConfig( object ): - always_conditional = ( 'pysam', 'ctypes', 'python_daemon' ) - - def __init__( self, config_file ): - if config_file is None: - self.config = None - else: - self.config = ConfigParser.ConfigParser() - if self.config.read( config_file ) == []: - raise Exception( "error: unable to read Galaxy config from %s" % config_file ) - - def check_conditional( self, egg_name ): - def check_pysam(): - # can't build pysam on solaris < 10 - plat = pkg_resources.get_platform().split( '-' ) - if plat[0] == 'solaris': - minor = plat[1].split('.')[1] - if int( minor ) < 10: - return False - return True - # If we're using require() we may not have a Galaxy config file, but if - # we're using require(), we don't care about conditionals. - if self.config is None: - return True - if egg_name == "pysqlite": - # SQLite is different since it can be specified in two config vars and defaults to True - try: - return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" ) - except: - return True - else: - try: - return { "psycopg2": - lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ), - "MySQL_python": - lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ), - "DRMAA_python": - lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","), - "drmaa": - lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","), - "pbs_python": - lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","), - "python_openid": - lambda: self.config.get( "app:main", "enable_openid" ), - "python_daemon": - lambda: sys.version_info[:2] >= ( 2, 5 ), - "pysam": - lambda: check_pysam(), - "PyRods": - lambda: self.config.get( "app:main", "object_store" ) == "irods" - }.get( egg_name, lambda: True )() - except: - return False - - -def string_as_bool( string ): - if str( string ).lower() in ( 'true', 'yes', 'on' ): - return True - else: - return False - - -def get_env(): - env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() ) - for dist in pkg_resources.find_distributions( eggs_dir, False ): - env.add( dist ) - return env -env = get_env() - - -def require( req_str ): - c = Crate( None ) - req = pkg_resources.Requirement.parse( req_str ) - if c.try_dependencies_from_env or not c.enable_eggs: - try: - return pkg_resources.working_set.require( req_str ) - except Exception as exc: - if not c.enable_eggs: - raise - log.info("%s not found in local environment, will try Galaxy egg: %s", (req_str, exc)) - # TODO: This breaks egg version requirements. Not currently a problem, but - # it could become one. - try: - return c[req.project_name].require() - except KeyError: - # not a galaxy-owned dependency - return pkg_resources.working_set.require( req_str ) - except EggNotFetchable, e: - raise EggNotFetchable( str( [ egg.name for egg in e.eggs ] ) ) -pkg_resources.require = require - - -def unpack_zipfile( filename, extract_dir, ignores=[] ): - z = zipfile.ZipFile(filename) - try: - for info in z.infolist(): - name = info.filename - mode = (info.external_attr >> 16L) & 0777 - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - target = os.path.join(extract_dir, *name.split('/')) - if not target: - continue - for ignore in ignores: - if ignore in name: - continue - if name.endswith('/'): - # directory - pkg_resources.ensure_directory(target) - else: - # file - pkg_resources.ensure_directory(target) - data = z.read(info.filename) - f = open(target, 'wb') - try: - f.write(data) - finally: - f.close() - del data - try: - if not os.path.islink(): - os.chmod(target, mode) - except: - pass - finally: - z.close() - - -def remove_file_or_path( f ): - if os.path.isdir( f ): - shutil.rmtree( f ) - else: - os.remove( f ) +require = pkg_resources.require diff --git a/lib/galaxy/eggs/dist.py b/lib/galaxy/eggs/dist.py deleted file mode 100644 index f44ff3021cd..00000000000 --- a/lib/galaxy/eggs/dist.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Manage Galaxy eggs -""" - -import os -import sys -import subprocess -from scramble import ScrambleEgg, ScrambleCrate, ScrambleFailure, galaxy_dir -from __init__ import Crate, CaseSensitiveConfigParser - -import logging -log = logging.getLogger( __name__ ) -log.addHandler( logging.NullHandler() ) - - -class DistScrambleEgg( ScrambleEgg ): - def set_dir( self ): - self.dir = os.path.join( galaxy_dir, 'dist-eggs', self.name ) - if not os.path.exists( self.dir ): - os.makedirs( self.dir ) - - @property - def path( self ): - # don't look for compatible eggs, look for exact matches - if os.path.exists( self.distribution.location ): - return self.distribution.location - return None - - def run_scramble_script( self ): - log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) - # subprocessed to sterilize the env - cmd = "ssh %s 'cd %s; PYTHONPATH= %s -ES %s'" % ( self.build_host, self.buildpath, self.python, 'scramble.py' ) - log.debug( '%s(): Executing:' % sys._getframe().f_code.co_name ) - log.debug( ' %s' % cmd ) - p = subprocess.Popen( args=cmd, shell=True ) - r = p.wait() - if r != 0: - raise ScrambleFailure( "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) - - def unpack_if_needed( self ): - return # do not unpack dist eggs - - -class DistScrambleCrate( ScrambleCrate ): - """ - Holds eggs with info on how to build them for distribution. - """ - dist_config_file = os.path.join( galaxy_dir, 'dist-eggs.ini' ) - - def __init__( self, galaxy_config_file, build_on='all' ): - self.dist_config = CaseSensitiveConfigParser() - self.build_on = build_on - ScrambleCrate.__init__( self, galaxy_config_file ) - - def parse( self ): - self.dist_config.read( DistScrambleCrate.dist_config_file ) - self.hosts = dict( self.dist_config.items( 'hosts' ) ) - self.groups = dict( self.dist_config.items( 'groups' ) ) - self.ignore = dict( self.dist_config.items( 'ignore' ) ) - self.platforms = self.get_platforms( self.build_on ) - self.noplatforms = self.get_platforms( 'noplatform' ) - Crate.parse( self ) - - def get_platforms( self, wanted ): - # find all the members of a group and process them - if wanted in self.groups: - platforms = [] - for name in self.groups[wanted].split(): - for platform in self.get_platforms( name ): - if platform not in platforms: - platforms.append( platform ) - return platforms - elif wanted in self.hosts: - return [ wanted ] - else: - raise Exception( "unknown platform: %s" % wanted ) - - def parse_egg_section( self, eggs, tags, full_platform=False ): - for name, version in eggs: - self.eggs[name] = [] - tag = dict( tags ).get( name, '' ) - url = '/'.join( ( self.repo, name ) ) - try: - sources = self.config.get( 'source', name ).split() - except: - sources = [] - try: - dependencies = self.config.get( 'dependencies', name ).split() - except: - dependencies = [] - if full_platform: - platforms = self.platforms - else: - platforms = self.noplatforms - for platform in platforms: - if name in self.ignore and platform in self.ignore[name].split(): - continue - egg = DistScrambleEgg( name, version, tag, url, platform, self ) - host_info = self.hosts[platform].split() - egg.build_host, egg.python = host_info[:2] - egg.sources = sources - egg.dependencies = dependencies - self.eggs[name].append( egg ) diff --git a/lib/galaxy/eggs/scramble.py b/lib/galaxy/eggs/scramble.py deleted file mode 100644 index 613d927d335..00000000000 --- a/lib/galaxy/eggs/scramble.py +++ /dev/null @@ -1,268 +0,0 @@ -""" -Manage Galaxy eggs -""" - -import os -import sys -import shutil -import subprocess -import urlparse -import urllib -from __init__ import Egg, Crate, URLRetriever, galaxy_dir, py, unpack_zipfile -from distutils.sysconfig import get_config_var - -import tarfile -import zipfile -arctypes = ( 'tar.gz', 'tgz', 'zip' ) - -import logging -log = logging.getLogger( __name__ ) -log.addHandler( logging.NullHandler() ) - - -class ScrambleFailure( Exception ): - def __init__( self, eggs, msg=None ): - if type( eggs ) in ( list, tuple ): - self.eggs = eggs - else: - self.eggs = [ eggs ] - self.msg = msg - - def __str__( self ): - return self.msg or ' '.join( self.eggs ) - - -class ScrambleEgg( Egg ): - """ - Contains information about scrambling eggs. - """ - scramble_dir = os.path.join( galaxy_dir, 'scripts', 'scramble' ) - archive_dir = os.path.join( scramble_dir, 'archives' ) - script_dir = os.path.join( scramble_dir, 'scripts' ) - build_dir = os.path.join( scramble_dir, 'build' ) - ez_setup = os.path.join( scramble_dir, 'lib', 'ez_setup.py' ) - ez_setup_url = 'http://peak.telecommunity.com/dist/ez_setup.py' - - def __init__( self, *args, **kwargs ): - Egg.__init__( self, *args, **kwargs ) - self.sources = [] - self.dependencies = [] - self.buildpath = None - self.source_path = None - self.py = py - self.build_host = None - self.python = sys.executable - - def scramble( self ): - if self.path: - log.warning( "%s(): Egg already exists, remove to force rebuild:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.path ) - return - self.fetch_source() - self.unpack_source() - self.copy_build_script() - if not os.path.exists( ScrambleEgg.ez_setup ): - URLRetriever().retrieve( ScrambleEgg.ez_setup_url, ScrambleEgg.ez_setup ) - self.run_scramble_script() - new_egg = os.path.join( self.buildpath, 'dist', os.path.basename( self.distribution.location ) ) - if not os.path.exists( new_egg ): - raise ScrambleFailure( self, "%s(): Egg build for %s did not appear to fail, but no egg found to copy from expected path:\n %s" % ( sys._getframe().f_code.co_name, self.name, new_egg ) ) - shutil.copyfile( new_egg, self.distribution.location ) - log.warning( "%s(): Copied egg to:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.distribution.location ) - self.unpack_if_needed() - self.remove_doppelgangers() - - # scramble helper methods - def get_tld( self, names ): - tld = names[0].split( os.path.sep, 1 )[0] - for name in names: - try: - assert tld == name.split( os.path.sep, 1 )[0] - except: - raise Exception( "%s(): Archive contains multiple top-level directories!" % sys._getframe().f_code.co_name ) - return tld - - def fetch_one( self, urls ): - """ - Fetches the first available archive out of a list. - """ - for url in urls: - file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) - if os.path.exists( file ): - log.warning( "%s(): Using existing source, remove to download again:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % file ) - return file - # if we don't have one, get one - for url in urls: - file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] ) - try: - log.debug( "%s(): Trying to fetch:" % sys._getframe().f_code.co_name ) - log.debug( " %s" % url ) - URLRetriever().retrieve( url, file + '.download' ) - shutil.move( file + '.download', file ) - log.debug( "%s(): Fetched to:" % sys._getframe().f_code.co_name ) - log.debug( " %s" % file ) - return file - except IOError, e: - if e[1] != 404: - raise - else: - return None - - def fetch_source( self ): - """ - Get egg (and dependent) source - """ - if not os.path.exists( ScrambleEgg.archive_dir ): - os.makedirs( ScrambleEgg.archive_dir ) - urls = [] - url_base = self.url + '/' + '-'.join( ( self.name, self.version ) ) - urls.extend( map( lambda x: '.'.join( ( url_base, x ) ), arctypes ) ) - if self.tag: - urls.extend( map( lambda x: '.'.join( ( url_base + self.tag, x ) ), arctypes ) ) - self.source_path = self.fetch_one( urls ) - if self.source_path is None: - raise Exception( "%s(): Couldn't find a suitable source archive for %s %s from %s" % ( sys._getframe().f_code.co_name, self.name, self.version, self.url ) ) - for url in self.sources: - if not urlparse.urlparse( url )[0]: - url = self.url + '/' + url.lstrip( '/' ) - urls = [ url ] - urls.extend( map( lambda x: '.'.join( ( url, x ) ), arctypes ) ) # allows leaving off the extension and we'll try to find one - file = self.fetch_one( urls ) - if file is None: - raise Exception( "%s(): Couldn't fetch extra source for %s, check path in %s. URL(s) attempted output above." % ( sys._getframe().f_code.co_name, self.name, Crate.config_file, ) ) - - def unpack_source( self ): - unpack_dir = os.path.join( ScrambleEgg.build_dir, self.platform ) - if not os.path.exists( unpack_dir ): - os.makedirs( unpack_dir ) - self.buildpath = os.path.join( unpack_dir, self.name ) - if os.path.exists( self.buildpath ): - log.warning( "%s(): Removing old build directory at:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.buildpath ) - shutil.rmtree( self.buildpath ) - if tarfile.is_tarfile( self.source_path ): - self.unpack_tar() - elif zipfile.is_zipfile( self.source_path ): - self.unpack_zip() - else: - raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, self.source_path ) ) - log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name ) - log.warning( " %s" % self.buildpath ) - - def unpack_zip( self ): - unpack_path = os.path.dirname( self.buildpath ) - tld = self.get_tld( zipfile.ZipFile( self.source_path, 'r' ).namelist() ) - unpack_zipfile( self.source_path, unpack_path, ( 'ez_setup', ) ) - os.rename( os.path.join( unpack_path, tld ), self.buildpath ) - - def unpack_tar( self ): - unpack_path = os.path.dirname( self.buildpath ) - t = tarfile.open( self.source_path, "r" ) - members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() ) - tld = self.get_tld( [ x.name for x in members ] ) - cur = os.getcwd() - os.chdir( unpack_path ) - for member in members: - try: - t.extract( member ) - except: - log.exception( "Could not extract %s", member ) - raise - t.close() - os.rename( tld, self.name ) - os.chdir( cur ) - - def copy_build_script( self ): - # will try: - # bx_python-py2.4-solaris-2.11-i86pc.py - # bx_python-py2.4-solaris.py - # bx_python-solaris-2.11-i86pc.py - # bx_python-solaris.py - # bx_python-py2.4.py - # bx_python.py - # generic.py - platform = self.platform.replace( '-ucs2', '' ).replace( '-ucs4', '' ) # ucs is unimportant here - build_scripts = ( - "%s-%s.py" % ( self.name, platform ), - "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[:2] ) ), - "%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[1:] ) ), - "%s-%s.py" % ( self.name, platform.split( '-' )[:2][-1] ), - "%s-%s.py" % ( self.name, platform.split( '-' )[0] ), - "%s.py" % self.name, - "generic.py" ) - for build_script in build_scripts: - build_script = os.path.join( ScrambleEgg.script_dir, build_script ) - if os.path.exists( build_script ): - log.warning( "%s(): Using build script %s" % ( sys._getframe().f_code.co_name, build_script ) ) - break - shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) ) - verfile = open( os.path.join( self.buildpath, ".galaxy_ver" ), "w" ) - verfile.write( self.version + '\n' ) - verfile.close() - if self.tag is not None: - tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" ) - tagfile.write( self.tag + '\n' ) - tagfile.close() - if self.dependencies: - depfile = open( os.path.join( self.buildpath, ".galaxy_deps" ), "w" ) - for dependency in self.dependencies: - depfile.write( dependency + '\n' ) - depfile.close() - - def run_scramble_script( self ): - log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name ) - # subprocessed to sterilize the env - cmd = "%s %s" % ( self.python, "scramble.py" ) - log.debug( '%s(): Executing in %s:' % ( sys._getframe().f_code.co_name, self.buildpath ) ) - log.debug( ' %s' % cmd ) - p = subprocess.Popen( args=cmd, shell=True, cwd=self.buildpath ) - r = p.wait() - if r != 0: - if sys.platform == 'sunos5' and get_config_var('CC').endswith('pycc') and not os.environ.get( 'PYCC_CC', None ): - log.error( "%s(): Your python interpreter was compiled with Sun's pycc" % sys._getframe().f_code.co_name ) - log.error( " pseudo-compiler. You may need to set PYCC_CC and PYCC_CXX in your" ) - log.error( " environment if your compiler is in a non-standard location." ) - raise ScrambleFailure( self, "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) ) - - -class ScrambleCrate( Crate ): - """ - Reads the eggs.ini file for use with scrambling eggs. - """ - def parse( self ): - Crate.parse( self ) - # get dependency sources - for egg in self.eggs.values(): - try: - egg.sources = self.config.get( "source", egg.name ).split() - except: - egg.sources = [] - try: - egg.dependencies = self.config.get( "dependencies", egg.name ).split() - except: - egg.dependencies = [] - - def parse_egg_section( self, *args, **kwargs ): - kwargs['egg_class'] = ScrambleEgg - Crate.parse_egg_section( self, *args, **kwargs ) - - def scramble( self, all=False ): - if all: - eggs = self.all_eggs - else: - eggs = self.config_eggs - eggs = filter( lambda x: x.name not in self.no_auto, eggs ) - failed = [] - for egg in eggs: - try: - egg.scramble() - except Exception, e: - failed.append( egg ) - last_exc = e - if failed: - if len( failed ) == 1: - raise last_exc # only 1 failure out of the crate, be more informative - else: - raise ScrambleFailure( failed ) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index ea708600dba..c21c1baf193 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -2,9 +2,6 @@ Custom exceptions for Galaxy """ -from galaxy import eggs -eggs.require( "Paste" ) - from ..exceptions import error_codes diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index fea999c0e46..259277d9fae 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -798,6 +798,10 @@ class JobWrapper( object ): self.__galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root return self.__galaxy_lib_dir + @property + def galaxy_virtual_env(self): + return os.environ.get('VIRTUAL_ENV', None) + # legacy naming get_job_runner = get_job_runner_url diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 87fa0ddf200..b936c5bb7a4 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -8,8 +8,6 @@ import logging import socket from galaxy.util import send_mail from galaxy.util.json import dumps -from galaxy import eggs -eggs.require( "MarkupSafe" ) from markupsafe import escape log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 0b2c0a5099a..42e7bedccf4 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -12,7 +12,6 @@ import subprocess from Queue import Queue, Empty -import galaxy.eggs import galaxy.jobs from galaxy.jobs.command_factory import build_command from galaxy import model @@ -36,6 +35,7 @@ JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s' JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" GALAXY_LIB_ADJUST_TEMPLATE = """GALAXY_LIB="%s"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi;""" +GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;""" class RunnerParams( ParamsWithSpecs ): @@ -254,12 +254,13 @@ class BaseJobRunner( object ): # so that long running set_meta()s can be canceled without having to reboot the server if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths: lib_adjust = GALAXY_LIB_ADJUST_TEMPLATE % job_wrapper.galaxy_lib_dir + venv = GALAXY_VENV_TEMPLATE % job_wrapper.galaxy_virtual_env external_metadata_script = job_wrapper.setup_external_metadata( output_fnames=job_wrapper.get_output_fnames(), set_extension=True, tmp_dir=job_wrapper.working_directory, # We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior kwds={ 'overwrite' : False } ) - external_metadata_script = "%s %s" % (lib_adjust, external_metadata_script) + external_metadata_script = "%s %s %s" % (lib_adjust, venv, external_metadata_script) if resolve_requirements: dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands() if dependency_shell_commands: @@ -275,20 +276,12 @@ class BaseJobRunner( object ): external_metadata_proc.wait() log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id ) - def _get_egg_env_opts(self): - env = [] - crate = galaxy.eggs.Crate() - for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'): - env.append('GALAXY_%s="%s"; export GALAXY_%s' % (opt.upper(), getattr(crate, opt), opt.upper())) - return env - def get_job_file(self, job_wrapper, **kwds): job_metrics = job_wrapper.app.job_metrics job_instrumenter = job_metrics.job_instrumenters[ job_wrapper.job_destination.id ] env_setup_commands = kwds.get( 'env_setup_commands', [] ) env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' ) - env_setup_commands.extend( self._get_egg_env_opts() or [] ) destination = job_wrapper.job_destination or {} envs = destination.get( "env", [] ) envs.extend( job_wrapper.environment_variables ) @@ -298,6 +291,7 @@ class BaseJobRunner( object ): options = dict( job_instrumenter=job_instrumenter, galaxy_lib=job_wrapper.galaxy_lib_dir, + galaxy_virtual_env=job_wrapper.galaxy_virtual_env, env_setup_commands=env_setup_commands, working_directory=os.path.abspath( job_wrapper.working_directory ), command=command_line, diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 944cd55f969..98b020ea5cd 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -1,6 +1,7 @@ """ Job control via the DRMAA API. """ +from __future__ import absolute_import import json import logging @@ -9,21 +10,26 @@ import string import subprocess import time -from galaxy import eggs +try: + import drmaa + DRMAA_IMPORT_MESSAGE = None +except (ImportError, RuntimeError) as exc: + drmaa = None + DRMAA_IMPORT_MESSAGE = ('The Python drmaa package is required to use this ' + 'feature, please install it or correct the ' + 'following error:\n%s: %s' % + (exc.__class__.__name__, str(exc))) + from galaxy import model from galaxy.jobs import JobDestination from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner from galaxy.util import asbool -eggs.require( "drmaa" ) - log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] -drmaa = None - DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'workingDirectory', 'jobName', 'email', 'project' ] @@ -36,8 +42,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): def __init__( self, app, nworkers, **kwargs ): """Start the job runner""" - - global drmaa + assert drmaa is not None, DRMAA_IMPORT_MESSAGE runner_param_specs = dict( drmaa_library_path=dict( map=str, default=os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ), @@ -57,10 +62,6 @@ class DRMAAJobRunner( AsynchronousJobRunner ): log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path ) os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path - # We foolishly named this file the same as the name exported by the drmaa - # library... 'import drmaa' imports itself. - drmaa = __import__( "drmaa" ) - # Subclasses may need access to state constants self.drmaa_job_states = drmaa.JobState diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index c01809de3bb..e4bd3432325 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -1,9 +1,20 @@ +from __future__ import absolute_import + import os import logging import time import traceback from datetime import timedelta +try: + import pbs + PBS_IMPORT_MESSAGE = None +except ImportError as exc: + pbs = None + PBS_IMPORT_MESSAGE = ('The Python pbs-python package is required to use ' + 'this feature, please install it or correct the ' + 'following error:\nImportError %s' % str(exc)) + from galaxy import model from galaxy import util from galaxy.util.bunch import Bunch @@ -11,25 +22,6 @@ from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size from galaxy.jobs import JobDestination from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner -import pkg_resources - -egg_message = """ - -The 'pbs' runner depends on 'pbs_python' which is not installed or not -configured properly. Galaxy's "scramble" system should make this installation -simple, please follow the instructions found at: - - http://wiki.galaxyproject.org/Admin/Config/Performance/Cluster - -Additional errors may follow: -%s -""" - -try: - pkg_resources.require( "pbs_python" ) - pbs = __import__( "pbs" ) -except Exception as e: - raise Exception( egg_message % str( e ) ) log = logging.getLogger( __name__ ) @@ -99,8 +91,7 @@ class PBSJobRunner( AsynchronousJobRunner ): def __init__( self, app, nworkers ): """Start the job runner """ # Check if PBS was importable, fail if not - if pbs is None: - raise Exception( "PBSJobRunner requires pbs_python which was not found" ) + assert pbs is not None, PBS_IMPORT_MESSAGE if app.config.pbs_application_server and app.config.outputs_to_working_directory: raise Exception( "pbs_application_server (file staging) and outputs_to_working_directory options are mutually exclusive" ) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 5d7119653a0..5820de9e4a1 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -20,6 +20,11 @@ if [ "$GALAXY_LIB" != "None" ]; then export PYTHONPATH fi $env_setup_commands +GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" +if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ + -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then + . "$GALAXY_VIRTUAL_ENV/bin/activate" +fi $instrument_pre_commands cd $working_directory $command diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index b481b06875b..8170443c279 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -15,6 +15,7 @@ GALAXY_SLOTS="1" REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path'] OPTIONAL_TEMPLATE_PARAMS = { 'galaxy_lib': None, + 'galaxy_virtual_env': None, 'headers': '', 'env_setup_commands': [], 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 9124995845c..c12ae7a513c 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -25,12 +25,7 @@ attribute change to a model object. # instead of the three separate classes. With no 'apparent' perfect scheme # I'm opting to just keep them separate. -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy - -pkg_resources.require("repoze.lru") # used by Routes -pkg_resources.require("Routes") import routes from galaxy import exceptions diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 2fed11bec06..32a85acad19 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,10 +1,7 @@ -import pkg_resources - import functools import os import urllib2 -pkg_resources.require('Beaker') from beaker.cache import CacheManager from beaker.util import parse_cache_config_options diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index e85d869aa63..c41f1a8c1af 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -6,8 +6,6 @@ created (or copied) by users over the course of an analysis. """ import operator -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import desc, asc from galaxy import model diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index fe83f399382..3cd940d3718 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -3,8 +3,6 @@ Manager and Serializer for libraries. """ import logging -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, false, not_, or_, true from sqlalchemy.orm.exc import MultipleResultsFound from sqlalchemy.orm.exc import NoResultFound diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 03456f7dde5..96e32ef45bb 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -2,8 +2,6 @@ Manager and Serializer for Users. """ -import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) import sqlalchemy from galaxy import model diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5d39f9f84d9..7a48740c9ad 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -18,8 +18,6 @@ from itertools import ifilter from string import Template from uuid import UUID, uuid4 -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, func, not_, or_, true, join, select from sqlalchemy.orm import joinedload, object_session, aliased from sqlalchemy.ext import hybrid diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 5387a582d90..c97a31d27d6 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -8,8 +8,6 @@ from sys import getsizeof from itertools import chain from collections import deque -from galaxy import eggs -eggs.require("SQLAlchemy") import sqlalchemy from galaxy import app diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 3e0b41d523a..a8cb71c411d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -5,8 +5,6 @@ are encapsulated here. import logging -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import ( and_, asc, Boolean, Column, DateTime, desc, false, ForeignKey, Integer, MetaData, not_, Numeric, select, String, Table, TEXT, true, Unicode, UniqueConstraint ) from sqlalchemy.ext.associationproxy import association_proxy diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index e0d3539f1bd..230293cb813 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -2,18 +2,9 @@ import sys import os.path import logging -from galaxy import eggs -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "decorator" ) # Required by sqlalchemy-migrate -eggs.require( "Tempita " ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy-migrate" ) - from migrate.versioning import repository, schema from sqlalchemy import create_engine -from galaxy.model.orm import dialect_to_egg from sqlalchemy import MetaData from sqlalchemy.exc import NoSuchTableError from sqlalchemy import Table @@ -36,18 +27,6 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" """ - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - eggs.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) diff --git a/lib/galaxy/model/orm/__init__.py b/lib/galaxy/model/orm/__init__.py index 9aad18b4c9f..f30f013a3a6 100644 --- a/lib/galaxy/model/orm/__init__.py +++ b/lib/galaxy/model/orm/__init__.py @@ -1,34 +1,3 @@ -import logging -import pkg_resources - -from galaxy import eggs -eggs.require('SQLAlchemy') - -log = logging.getLogger( __name__ ) - -dialect_to_egg = { - "sqlite": "pysqlite>=2", - "postgres": "psycopg2", - "postgresql": "psycopg2", - "mysql": "MySQL_python" -} - - -def load_egg_for_url( url ): - # Load the appropriate db module - dialect = __guess_dialect_for_url( url ) - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module's in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - - -def __guess_dialect_for_url( url ): - return (url.split(':', 1))[0] +""" +galaxy.model.orm - ORM-related functionality +""" diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py index 7ddaafd4861..4183e985e54 100644 --- a/lib/galaxy/model/orm/engine_factory.py +++ b/lib/galaxy/model/orm/engine_factory.py @@ -2,12 +2,9 @@ import logging log = logging.getLogger( __name__ ) from sqlalchemy import create_engine -from galaxy.model.orm import load_egg_for_url def build_engine(url, engine_options, database_query_profiling_proxy=False, trace_logger=None): - load_egg_for_url( url ) - # Should we use the logging proxy? if database_query_profiling_proxy: import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy diff --git a/lib/galaxy/model/orm/logging_connection_proxy.py b/lib/galaxy/model/orm/logging_connection_proxy.py index 60bbd1762e3..985c0a2091d 100644 --- a/lib/galaxy/model/orm/logging_connection_proxy.py +++ b/lib/galaxy/model/orm/logging_connection_proxy.py @@ -4,8 +4,6 @@ import os import threading import time -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy.interfaces import ConnectionProxy log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index fb4c332d5a4..86d5dc02852 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -4,19 +4,8 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...). import logging import os.path -from galaxy import eggs - -eggs.require( "decorator" ) -eggs.require( "Tempita" ) -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # required by sqlalchemy-migrate -eggs.require( "sqlalchemy_migrate" ) - from galaxy.util.properties import load_app_properties -from galaxy.model.orm import dialect_to_egg -import pkg_resources log = logging.getLogger( __name__ ) @@ -48,21 +37,6 @@ DATABASE = { } -def require_dialect_egg( db_url ): - dialect = ( db_url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - pkg_resources.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - - def read_config_file_arg( argv, default, old_default ): if '-c' in argv: pos = argv.index( '-c' ) @@ -132,5 +106,4 @@ def get_config( argv, cwd=None ): else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file - require_dialect_egg( db_url ) return dict(db_url=db_url, repo=repo, config_file=config_file, database=database) diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index 229949a4a56..6aba6b7ec0d 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -27,8 +27,6 @@ select * from history where name='Unnamed history' import logging import re -from galaxy import eggs -eggs.require("Parsley") import parsley from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index 006fb974ea7..c7d16395886 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -2,14 +2,6 @@ import sys import os.path import logging -from galaxy import eggs -eggs.require( "SQLAlchemy" ) -eggs.require( "six" ) # Required by sqlalchemy-migrate -eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate -eggs.require( "decorator" ) # Required by sqlalchemy-migrate -eggs.require( "Tempita " ) # Required by sqlalchemy-migrate -eggs.require( "sqlalchemy-migrate" ) - # from sqlalchemy import * from sqlalchemy import create_engine from sqlalchemy import MetaData @@ -18,8 +10,6 @@ from sqlalchemy import Table from sqlalchemy.exc import NoSuchTableError from migrate.versioning import repository, schema -from galaxy.model.orm import dialect_to_egg - log = logging.getLogger( __name__ ) # path relative to galaxy @@ -30,18 +20,6 @@ migrate_repository = repository.Repository( migrate_repository_directory ) def create_or_verify_database( url, engine_options={}, app=None ): """ """ - dialect = ( url.split( ':', 1 ) )[0] - try: - egg = dialect_to_egg[dialect] - try: - eggs.require( egg ) - log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) - except: - # If the module is in the path elsewhere (i.e. non-egg), it'll still load. - log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) - except KeyError: - # Let this go, it could possibly work with db's we don't support - log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4754156ad5b..5567953b699 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -15,17 +15,14 @@ from posixpath import dirname as path_dirname from galaxy.exceptions import ObjectNotFound from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs -try: - import galaxy.eggs - galaxy.eggs.require( 'PyRods' ) -except Exception: - pass try: import irods except ImportError: irods = None -NO_PYRODS_ERROR_MESSAGE = "IRODS object store configured, but no PyRods dependency available. Please install and properly configure PyRods or modify object store configuration." + +IRODS_IMPORT_MESSAGE = ('The Python irods package is required to use this ' + 'feature, please install it') log = logging.getLogger( __name__ ) @@ -35,9 +32,8 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): Galaxy object store based on iRODS """ def __init__( self, config, file_path=None, extra_dirs=None ): - if irods is None: - raise Exception(NO_PYRODS_ERROR_MESSAGE) super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs ) + assert irods is not None, IRODS_IMPORT_MESSAGE self.cache_path = config.object_store_cache_path self.default_resource = config.irods_default_resource or None diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py index 257740a07e2..ac922e3447f 100644 --- a/lib/galaxy/objectstore/s3_multipart_upload.py +++ b/lib/galaxy/objectstore/s3_multipart_upload.py @@ -14,12 +14,6 @@ import subprocess from multiprocessing.pool import IMapIterator -try: - from galaxy import eggs - eggs.require('boto') -except ImportError: - pass - try: import boto from boto.s3.connection import S3Connection diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 918da8aefb1..3ed455cea01 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -5,16 +5,9 @@ reloading the toolbox, etc., across multiple processes. import logging import threading -import sys import galaxy.queues -from galaxy import eggs, util -eggs.require('anyjson') -if sys.version_info < (2, 7, 0): - # Kombu requires importlib and ordereddict to function under Python 2.6. - eggs.require('importlib') - eggs.require('ordereddict') -eggs.require('kombu') +from galaxy import util from kombu import Connection from kombu.mixins import ConsumerMixin diff --git a/lib/galaxy/queues.py b/lib/galaxy/queues.py index 6d5813d3faf..0779c3dcb0d 100644 --- a/lib/galaxy/queues.py +++ b/lib/galaxy/queues.py @@ -4,18 +4,8 @@ All message queues used by Galaxy """ -import sys - -from galaxy import eggs from galaxy.config import process_is_uwsgi -eggs.require('PyYAML') -eggs.require('anyjson') -if sys.version_info < (2, 7, 0): - # Kombu requires importlib and ordereddict to function in Python 2.6. - eggs.require('importlib') - eggs.require('ordereddict') -eggs.require('kombu') from kombu import Exchange, Queue, Connection ALL_CONTROL = "control.*" diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index 601fa20f276..0e9b7f2450b 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -7,8 +7,6 @@ import operator import socket from datetime import datetime, timedelta -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, false, or_, not_ from sqlalchemy.orm import eagerload_all diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 42a9eeec5c5..cf1ee9eff5d 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -12,13 +12,6 @@ import threading import urllib from datetime import datetime -from galaxy import eggs - -eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako -eggs.require( "Mako" ) -eggs.require( "Paste" ) -eggs.require( "SQLAlchemy >= 0.4" ) - from cgi import FieldStorage from xml.etree import ElementTree from mako.template import Template diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 4f3100c36be..76be42faca6 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -5,8 +5,6 @@ import subprocess import tempfile from cgi import FieldStorage -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy.orm import eagerload_all from galaxy import datatypes, util diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index fea5325f39a..56e29269052 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,8 +5,6 @@ import os import shutil import tempfile -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy.orm import eagerload, eagerload_all from galaxy import model diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index 2a2b089bfbe..25c553c03d1 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -1,11 +1,5 @@ from __future__ import absolute_import -try: - from galaxy import eggs - eggs.require("PyYAML") -except ImportError: - pass - import yaml from .yaml import YamlToolSource diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 3293ec09c54..a4d5721bc4a 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -2,10 +2,8 @@ Module for building and searching the index of tools installed within this Galaxy. """ -from galaxy import eggs from galaxy.web.framework.helpers import to_unicode -eggs.require( "Whoosh" ) from whoosh.filedb.filestore import RamStorage from whoosh.fields import Schema, STORED, TEXT from whoosh.scoring import BM25F diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 22c393af709..1238cb75f11 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -5,10 +5,7 @@ import string import tarfile import tempfile -from galaxy import eggs -eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import and_ -eggs.require( "MarkupSafe" ) from markupsafe import escape from galaxy.model.item_attrs import Dictifiable diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py index b40730f09bf..008c5d9d0c8 100644 --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -8,8 +8,7 @@ import os import string import sys import tempfile -import pkg_resources -pkg_resources.require( "bx-python" ) + import bx.align.maf import bx.intervals import bx.interval_index_file diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 09235bcfe82..e04c973f07c 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -33,15 +33,11 @@ from os.path import relpath from hashlib import md5 from itertools import izip -from galaxy import eggs - -eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 from xml.etree import ElementTree, ElementInclude -eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English diff --git a/lib/galaxy/util/log/fluent_log.py b/lib/galaxy/util/log/fluent_log.py index a8d1586e6e9..e1e6224e74a 100644 --- a/lib/galaxy/util/log/fluent_log.py +++ b/lib/galaxy/util/log/fluent_log.py @@ -4,23 +4,19 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector import threading -try: - import galaxy.eggs - galaxy.eggs.require( "fluent-logger" ) - galaxy.eggs.require( "msgpack_python" ) -except Exception: - pass - try: from fluent.sender import FluentSender except ImportError: FluentSender = None +FLUENT_IMPORT_MESSAGE = ('The Python fluent package is required to use this ' + 'feature, please install it') + + class FluentTraceLogger( object ): def __init__( self, name, host='localhost', port=24224 ): - if FluentSender is None: - raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.") + assert FluentSender is not None, FLUENT_IMPORT_MESSAGE self.lock = threading.Lock() self.thread_local = threading.local() self.name = name diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index b63d0c203d1..c03633692cc 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -1,11 +1,5 @@ from xml.etree import ElementTree -try: - from galaxy import eggs - eggs.require('PyYAML') -except Exception: - # If not in Galaxy, ignore this. - pass try: import yaml except ImportError: diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index 88cde7c8678..8e50f8e34d9 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -1,6 +1,3 @@ -import pkg_resources -pkg_resources.require( "Cheetah" ) - from Cheetah.Template import Template diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index 9d849a0e4f7..a8706dfe6f1 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -9,14 +9,12 @@ import random import re import sys -from galaxy import eggs -eggs.require('numpy') # noqa -eggs.require('bx-python') # noqa +import pysam + from bx.interval_index_file import Indexes from bx.bbi.bigbed_file import BigBedFile from bx.bbi.bigwig_file import BigWigFile -eggs.require('pysam') # noqa -from pysam import csamtools, ctabix +from pysam import ctabix from galaxy.datatypes.interval import Bed, Gff, Gtf from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes @@ -849,11 +847,11 @@ class BamDataProvider( GenomeDataProvider, FilterableMixin ): """ # Open current BAM file using index. - bamfile = csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb', - index_filename=self.converted_dataset.file_name ) + bamfile = pysam.AlignmentFile( filename=self.original_dataset.file_name, mode='rb', + index_filename=self.converted_dataset.file_name ) # TODO: write headers as well? - new_bamfile = csamtools.Samfile( template=bamfile, filename=filename, mode='wb' ) + new_bamfile = pysam.AlignmentFile( template=bamfile, filename=filename, mode='wb' ) for region in regions: # Write data from region. @@ -881,7 +879,7 @@ class BamDataProvider( GenomeDataProvider, FilterableMixin ): def open_data_file( self ): # Attempt to open the BAM file with index - return csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb', + return pysam.AlignmentFile( filename=self.original_dataset.file_name, mode='rb', index_filename=self.converted_dataset.file_name ) def get_iterator( self, data_file, chrom, start, end, **kwargs ): diff --git a/lib/galaxy/visualization/plugins/plugin.py b/lib/galaxy/visualization/plugins/plugin.py index 16d75c250a2..cd354dd87aa 100644 --- a/lib/galaxy/visualization/plugins/plugin.py +++ b/lib/galaxy/visualization/plugins/plugin.py @@ -6,9 +6,6 @@ from a query string and render a webpage based on those data. import os import copy -import pkg_resources -pkg_resources.require( 'MarkupSafe' ) -pkg_resources.require( 'Mako' ) import mako from galaxy.managers import api_keys diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index e13c0e1f1fb..f9be1af8978 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -5,9 +5,6 @@ import logging import operator import re -import pkg_resources -pkg_resources.require("SQLAlchemy >= 0.4") - from sqlalchemy import true from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError diff --git a/lib/galaxy/web/base/controllers/admin.py b/lib/galaxy/web/base/controllers/admin.py index a44ee1bf884..37821a3415c 100644 --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -3,8 +3,6 @@ import os from datetime import datetime, timedelta from string import punctuation as PUNCTUATION -from galaxy import eggs -eggs.require('SQLAlchemy') from sqlalchemy import and_, false, func, or_ import galaxy.queue_worker diff --git a/lib/galaxy/web/base/pluginframework.py b/lib/galaxy/web/base/pluginframework.py index c2f323fa45b..f52e5f08023 100644 --- a/lib/galaxy/web/base/pluginframework.py +++ b/lib/galaxy/web/base/pluginframework.py @@ -16,9 +16,6 @@ from galaxy import util from galaxy.util import odict from galaxy.util import bunch -import pkg_resources -pkg_resources.require( 'MarkupSafe' ) -pkg_resources.require( 'Mako' ) import mako.lookup import logging log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index b4e3f5503b3..a44e749c94d 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -2,8 +2,5 @@ Galaxy web application framework """ -from galaxy import eggs -eggs.require( "amqp" ) - import base url_for = base.routes.url_for diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 845dda45e6c..85db7e3df42 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -9,13 +9,6 @@ import socket import tarfile import types -import pkg_resources - -pkg_resources.require("Paste") -pkg_resources.require("repoze.lru") # used by Routes -pkg_resources.require("Routes") -pkg_resources.require("WebOb") - import routes import webob diff --git a/lib/galaxy/web/framework/decorators.py b/lib/galaxy/web/framework/decorators.py index 0f5f6d2a8c7..9830bfbb21f 100644 --- a/lib/galaxy/web/framework/decorators.py +++ b/lib/galaxy/web/framework/decorators.py @@ -2,8 +2,6 @@ import inspect from traceback import format_exc from functools import wraps -from galaxy import eggs -eggs.require( "Paste" ) import paste.httpexceptions from galaxy.web.framework import url_for diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index d45c1a729dd..f84facaaa39 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -4,16 +4,11 @@ Galaxy web framework helpers import time from datetime import datetime, timedelta -from galaxy import eggs from galaxy.util import hash_util from galaxy.util.json import safe_dumps as dumps # noqa (used by mako templates) -eggs.require( "MarkupSafe" ) # required by WebHelpers -eggs.require( "WebHelpers" ) from webhelpers import date from webhelpers.html.tags import stylesheet_link, javascript_link -eggs.require( "repoze.lru" ) # used by Routes -eggs.require( "Routes" ) from routes import url_for server_starttime = int(time.time()) diff --git a/lib/galaxy/web/framework/middleware/profile.py b/lib/galaxy/web/framework/middleware/profile.py index bb0dd9da185..d4fa6bcfb43 100644 --- a/lib/galaxy/web/framework/middleware/profile.py +++ b/lib/galaxy/web/framework/middleware/profile.py @@ -5,21 +5,11 @@ information at the bottom of each page. import threading import cgi +import cProfile +import pstats + from paste import response -try: - # Included in Python 2.5 - import cProfile -except: - try: - # Included in lsprof package for Python 2.4 - import pkg_resources - pkg_resources.require( "lsprof" ) - import cProfile - except: - cProfile = None - -import pstats template = """