mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #428 from natefoo/smash_eggs
Replace eggs with wheels
This commit is contained in:
Dannon Baker
@@ -1,4 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
if [ -d .venv ];
|
||||
then
|
||||
printf "Activating virtualenv at %s/.venv\n" $(pwd)
|
||||
. .venv/bin/activate
|
||||
fi
|
||||
|
||||
cd `dirname $0`
|
||||
python ./scripts/create_db.py $@
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
;
|
||||
; Config for building eggs for distribution (via a site such as
|
||||
; eggs.galaxyproject.org). Probably only useful to members of the Galaxy Team
|
||||
; building eggs for distribution. This file is used by
|
||||
; scripts/dist-scramble.py
|
||||
;
|
||||
; More information: http://wiki.galaxyproject.org/Admin/Config/Eggs
|
||||
;
|
||||
|
||||
[hosts]
|
||||
py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-i686-ucs2/bin/python2.6
|
||||
py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-i686-ucs4/bin/python2.6
|
||||
py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs2/bin/python2.6
|
||||
py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs4/bin/python2.6
|
||||
py2.7-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-i686-ucs2/bin/python2.7
|
||||
py2.7-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-i686-ucs4/bin/python2.7
|
||||
py2.7-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs2/bin/python2.7
|
||||
py2.7-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs4/bin/python2.7
|
||||
py2.6-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.6
|
||||
py2.7-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.7
|
||||
;
|
||||
; these hosts are used to build eggs with no C extensions
|
||||
py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.6-linux-x86_64-ucs4/bin/python2.6
|
||||
py2.7 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/py2.7-linux-x86_64-ucs4/bin/python2.7
|
||||
|
||||
[groups]
|
||||
py2.6-linux-i686 = py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4
|
||||
py2.6-linux-x86_64 = py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4
|
||||
py2.7-linux-i686 = py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4
|
||||
py2.7-linux-x86_64 = py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4
|
||||
py2.6-linux = py2.6-linux-i686 py2.6-linux-x86_64
|
||||
py2.7-linux = py2.7-linux-i686 py2.7-linux-x86_64
|
||||
linux-i686 = py2.6-linux-i686 py2.7-linux-i686
|
||||
linux-x86_64 = py2.6-linux-x86_64 py2.7-linux-x86_64
|
||||
linux = linux-i686 linux-x86_64
|
||||
py2.6-macosx = py2.6-macosx-10.6-intel-ucs2
|
||||
py2.7-macosx = py2.7-macosx-10.6-intel-ucs2
|
||||
macosx = py2.6-macosx py2.7-macosx
|
||||
py2.6-all = py2.6-linux py2.6-macosx
|
||||
py2.7-all = py2.7-linux py2.7-macosx
|
||||
|
||||
; the 'all' key is used internally by the build system to specify which hosts
|
||||
; to build on when no hosts are specified on the dist-eggs.py command line.
|
||||
all = linux macosx
|
||||
|
||||
; the 'noplatform' key, likewise, is for which build hosts should be used when
|
||||
; building pure python (noplatform) eggs.
|
||||
noplatform = py2.6 py2.7
|
||||
|
||||
; don't build these eggs on these platforms:
|
||||
[ignore]
|
||||
;ctypes =
|
||||
@@ -1,96 +0,0 @@
|
||||
;
|
||||
; Galaxy egg version configuration
|
||||
;
|
||||
; This file is version controlled and should not be edited by hand!
|
||||
; For more information, see:
|
||||
; https://wiki.galaxyproject.org/Admin/Config/Eggs
|
||||
;
|
||||
|
||||
[general]
|
||||
repository = http://eggs.galaxyproject.org
|
||||
; these eggs must be scrambled for your local environment
|
||||
no_auto = pbs_python
|
||||
|
||||
[eggs:platform]
|
||||
bx_python = 0.7.2
|
||||
Cheetah = 2.2.2
|
||||
MarkupSafe = 0.12
|
||||
mercurial = 3.2.4
|
||||
MySQL_python = 1.2.3c1
|
||||
PyRods = 3.2.4
|
||||
numpy = 1.6.0
|
||||
pbs_python = 4.3.5
|
||||
psycopg2 = 2.5.1
|
||||
pycrypto = 2.5
|
||||
pysam = 0.4.2
|
||||
pysqlite = 2.5.6
|
||||
python_lzo = 1.08_2.03_static
|
||||
PyYAML = 3.10
|
||||
SQLAlchemy = 1.0.8
|
||||
; msgpack_python = 0.2.4
|
||||
|
||||
[eggs:noplatform]
|
||||
amqp = 1.4.6
|
||||
anyjson = 0.3.3
|
||||
Beaker = 1.4
|
||||
bioblend = 0.5.2
|
||||
boto = 2.27.0
|
||||
decorator = 3.1.2
|
||||
docutils = 0.7
|
||||
drmaa = 0.7.6
|
||||
Fabric = 1.7.0
|
||||
importlib = 1.0.3
|
||||
kombu = 3.0.24
|
||||
lrucache = 0.2
|
||||
Mako = 0.4.1
|
||||
mock = 1.0.1
|
||||
nose = 1.3.6
|
||||
NoseHTML = 0.4.1
|
||||
NoseTestDiff = 0.1
|
||||
ordereddict = 1.1
|
||||
paramiko = 1.11.1
|
||||
Parsley = 1.1
|
||||
Paste = 1.7.5.1
|
||||
PasteDeploy = 1.5.0
|
||||
pexpect = 2.4
|
||||
poster = 0.8.1
|
||||
python_openid = 2.2.5
|
||||
python_daemon = 1.5.5
|
||||
pytz = 2013.9
|
||||
repoze.lru = 0.6
|
||||
requests = 2.2.1
|
||||
Routes = 2.1
|
||||
simplejson = 2.1.1
|
||||
six = 1.9.0
|
||||
sqlalchemy_migrate = 0.9.6
|
||||
sqlparse = 0.1.14
|
||||
ssh = 1.7.14
|
||||
SVGFig = 1.1.6
|
||||
Tempita = 0.5.1
|
||||
twill = 0.9
|
||||
WebError = 0.8a
|
||||
WebHelpers = 1.3
|
||||
WebOb = 0.8.5
|
||||
wsgiref = 0.1.2
|
||||
Babel = 1.3
|
||||
wchartype = 0.1
|
||||
Whoosh = 2.4.1
|
||||
; fluent_logger = 0.3.3
|
||||
raven = 3.1.8
|
||||
|
||||
; extra version information
|
||||
[tags]
|
||||
psycopg2 = _9.2.4_static
|
||||
pysqlite = _3.6.17_static
|
||||
MySQL_python = _5.1.41_static
|
||||
pysam = _kanwei_b10f6e722e9a
|
||||
|
||||
; dependency source urls, necessary for scrambling. for an explanation, see
|
||||
; the wiki page above
|
||||
[source]
|
||||
MySQL_python = mysql-5.1.41
|
||||
psycopg2 = postgresql-9.2.4
|
||||
pysqlite = sqlite-amalgamation-3_6_17
|
||||
|
||||
[dependencies]
|
||||
bx_python = numpy
|
||||
+2
-75
@@ -2,82 +2,9 @@
|
||||
Galaxy root package -- this is a namespace package.
|
||||
"""
|
||||
|
||||
__import__( "pkg_resources" ).declare_namespace( __name__ )
|
||||
from pkgutil import extend_path
|
||||
__path__ = extend_path(__path__, __name__)
|
||||
|
||||
import sys
|
||||
import platform
|
||||
|
||||
import pkg_resources
|
||||
|
||||
|
||||
# patch get_platform() for better ABI recognition
|
||||
def _get_build_platform():
|
||||
plat = pkg_resources._get_build_platform()
|
||||
if sys.platform == 'darwin':
|
||||
# Assert OS X version is new enough
|
||||
current_minor_ver = int( platform.mac_ver()[0].split( '.' )[1] )
|
||||
assert current_minor_ver >= 6, 'Galaxy is not compatible with Mac OS X < 10.6 (your version is %s)' % platform.mac_ver()[0]
|
||||
# Python build target may be even older, but this will prevent it from
|
||||
# fetching our 10.6+ eggs, so fix if necessary (newer versions will be
|
||||
# fine)
|
||||
plat_split = plat.split( '-' )
|
||||
build_minor_ver = int( plat_split[1].split( '.' )[1] )
|
||||
if build_minor_ver < 6:
|
||||
plat_split[1] = '10.6'
|
||||
# Our intel (dual arch) eggs will work fine on single-arch builds
|
||||
if plat_split[-1] in ( 'i386', 'x86_64' ):
|
||||
plat_split[-1] = 'intel'
|
||||
# Ditto universal (if you're not on PPC)
|
||||
if plat_split[-1] == 'universal' and platform.processor() != 'powerpc':
|
||||
plat_split[-1] = 'intel'
|
||||
plat = '-'.join( plat_split )
|
||||
elif sys.platform == "linux2" and sys.maxint < 2 ** 31 and plat.endswith( '-x86_64' ):
|
||||
# 32 bit Python on 64 bit Linux
|
||||
plat = plat.replace( '-x86_64', '-i686' )
|
||||
if not (plat.endswith('-ucs2') or plat.endswith('-ucs4')):
|
||||
if sys.maxunicode > 2 ** 16:
|
||||
plat += '-ucs4'
|
||||
else:
|
||||
plat += '-ucs2'
|
||||
return plat
|
||||
try:
|
||||
assert pkg_resources._get_build_platform
|
||||
except:
|
||||
pkg_resources._get_build_platform = pkg_resources.get_build_platform
|
||||
pkg_resources.get_build_platform = _get_build_platform
|
||||
pkg_resources.get_platform = _get_build_platform
|
||||
|
||||
|
||||
# patch to insert eggs at the beginning of sys.path instead of at the end
|
||||
def _insert_on(self, path, loc=None):
|
||||
"""Insert self.location in path before its nearest parent directory"""
|
||||
|
||||
loc = loc or self.location
|
||||
if not loc:
|
||||
return
|
||||
|
||||
nloc = pkg_resources._normalize_cached(loc)
|
||||
npath = [(p and pkg_resources._normalize_cached(p) or p) for p in path]
|
||||
|
||||
if path is sys.path:
|
||||
self.check_version_conflict()
|
||||
path.insert(0, loc)
|
||||
|
||||
# remove dups
|
||||
while 1:
|
||||
try:
|
||||
np = npath.index(nloc, 1)
|
||||
except ValueError:
|
||||
break
|
||||
else:
|
||||
del npath[np], path[np]
|
||||
|
||||
return
|
||||
try:
|
||||
assert pkg_resources.Distribution._insert_on
|
||||
except:
|
||||
pkg_resources.Distribution._insert_on = pkg_resources.Distribution.insert_on
|
||||
pkg_resources.Distribution.insert_on = _insert_on
|
||||
|
||||
# compat: BadZipFile introduced in Python 2.7
|
||||
import zipfile
|
||||
|
||||
@@ -14,7 +14,6 @@ import string
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import timedelta
|
||||
from galaxy import eggs
|
||||
from galaxy.exceptions import ConfigurationError
|
||||
from galaxy.util import listify
|
||||
from galaxy.util import string_as_bool
|
||||
@@ -729,7 +728,6 @@ def configure_logging( config ):
|
||||
root.addHandler( handler )
|
||||
# If sentry is configured, also log to it
|
||||
if config.sentry_dsn:
|
||||
eggs.require( "raven" )
|
||||
from raven.handlers.logging import SentryHandler
|
||||
sentry_handler = SentryHandler( config.sentry_dsn )
|
||||
sentry_handler.setLevel( logging.WARN )
|
||||
|
||||
@@ -8,11 +8,10 @@ import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import warnings
|
||||
import zipfile
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
import pysam
|
||||
|
||||
from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE
|
||||
|
||||
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter
|
||||
@@ -20,11 +19,6 @@ from galaxy.datatypes import metadata
|
||||
from galaxy.util import nice_size, sqlite
|
||||
from . import data, dataproviders
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter( "ignore" )
|
||||
eggs.require( "pysam" )
|
||||
from pysam import csamtools
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -340,7 +334,7 @@ class Bam( Binary ):
|
||||
os.unlink( stderr_name )
|
||||
# Now use pysam with BAI index to determine additional metadata
|
||||
try:
|
||||
bam_file = csamtools.Samfile( filename=dataset.file_name, mode='rb', index_filename=index_file.file_name )
|
||||
bam_file = pysam.AlignmentFile( filename=dataset.file_name, mode='rb', index_filename=index_file.file_name )
|
||||
dataset.metadata.reference_names = list( bam_file.references )
|
||||
dataset.metadata.reference_lengths = list( bam_file.lengths )
|
||||
dataset.metadata.bam_header = bam_file.header
|
||||
@@ -348,6 +342,8 @@ class Bam( Binary ):
|
||||
dataset.metadata.sort_order = dataset.metadata.bam_header.get( 'HD', {} ).get( 'SO', None )
|
||||
dataset.metadata.bam_version = dataset.metadata.bam_header.get( 'HD', {} ).get( 'VN', None )
|
||||
except:
|
||||
# Per Dan, don't log here because doing so will cause datasets that
|
||||
# fail metadata to end in the error state
|
||||
pass
|
||||
|
||||
def sniff( self, filename ):
|
||||
|
||||
@@ -4,10 +4,6 @@ from __future__ import division
|
||||
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
|
||||
eggs.require('numpy') # noqa
|
||||
eggs.require('bx-python') # noqa
|
||||
from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict
|
||||
|
||||
BLOCK_SIZE = 100
|
||||
|
||||
@@ -9,9 +9,7 @@ import optparse
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "pysam" )
|
||||
import ctabix
|
||||
from pysam import ctabix
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -11,8 +11,6 @@ from __future__ import division
|
||||
import fileinput
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.interval_index_file import Indexes
|
||||
|
||||
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GenomicInterval, GFFReaderWrapper
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# Dan Blankenberg
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
import bx.intervals.io
|
||||
|
||||
assert sys.version_info[:2] >= ( 2, 4 )
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# Dan Blankenberg
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
import bx.intervals.io
|
||||
|
||||
assert sys.version_info[:2] >= ( 2, 4 )
|
||||
|
||||
@@ -11,8 +11,6 @@ import tempfile
|
||||
from bisect import bisect
|
||||
from os import environ
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.cookbook import doc_optparse
|
||||
from bx.intervals import io
|
||||
|
||||
|
||||
@@ -15,8 +15,6 @@ and symbols are sorted in lexigraphical order.
|
||||
'''
|
||||
import optparse
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.tabular.io import Comment
|
||||
|
||||
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFReaderWrapper, read_unordered_gtf
|
||||
|
||||
@@ -12,8 +12,6 @@ usage: %prog <options> in_file out_file
|
||||
from __future__ import division
|
||||
|
||||
import optparse
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.interval_index_file import Indexes
|
||||
|
||||
|
||||
|
||||
@@ -10,9 +10,7 @@ import optparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "pysam" )
|
||||
import ctabix
|
||||
from pysam import ctabix
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# Dan Blankenberg
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
import bx.align.maf
|
||||
|
||||
from galaxy.tools.util import maf_utilities
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# Dan Blankenberg
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
import bx.align.maf
|
||||
|
||||
from galaxy.tools.util import maf_utilities
|
||||
|
||||
@@ -9,8 +9,6 @@ usage: %prog <options> in_file out_file
|
||||
from __future__ import division
|
||||
|
||||
import optparse
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.interval_index_file import Indexes
|
||||
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ from __future__ import division
|
||||
|
||||
import optparse
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.interval_index_file import Indexes
|
||||
|
||||
import galaxy_utils.sequence.vcf
|
||||
|
||||
@@ -9,9 +9,7 @@ usage: %prog in_file out_file
|
||||
"""
|
||||
import optparse
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "pysam" )
|
||||
import ctabix
|
||||
from pysam import ctabix
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
@@ -4,9 +4,6 @@ from __future__ import division
|
||||
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('numpy') # noqa
|
||||
eggs.require('bx-python') # noqa
|
||||
from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict
|
||||
from bx.arrays.wiggle import WiggleReader
|
||||
|
||||
|
||||
@@ -8,8 +8,6 @@ and fixedStep wiggle lines.
|
||||
"""
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
import bx.wiggle
|
||||
|
||||
from galaxy.tools.exception_handling import UCSCOutWrapper, UCSCLimitException
|
||||
|
||||
@@ -6,7 +6,6 @@ Coverage datatypes
|
||||
import logging
|
||||
import math
|
||||
|
||||
from galaxy import eggs
|
||||
from galaxy.datatypes import metadata
|
||||
from galaxy.datatypes.metadata import MetadataElement
|
||||
from galaxy.datatypes.tabular import Tabular
|
||||
@@ -28,7 +27,6 @@ class LastzCoverage( Tabular ):
|
||||
Assumes we have a numpy file.
|
||||
"""
|
||||
# Maybe if we import here people will still be able to use Galaxy when numpy kills it
|
||||
eggs.require("numpy>=1.2.1")
|
||||
import numpy
|
||||
|
||||
range = end - start
|
||||
|
||||
@@ -17,8 +17,6 @@ from galaxy.util.sanitize_html import sanitize_html
|
||||
|
||||
import dataproviders
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "Paste" )
|
||||
import paste
|
||||
|
||||
XSS_VULNERABLE_MIME_TYPES = [
|
||||
|
||||
@@ -14,8 +14,6 @@ import external
|
||||
from galaxy.util import sqlite
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( 'bx-python' )
|
||||
from bx import seq as bx_seq
|
||||
from bx import wiggle as bx_wig
|
||||
from bx import bbi as bx_bbi
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
from galaxy import eggs
|
||||
eggs.require( "pycrypto" )
|
||||
from Crypto.Cipher import Blowfish
|
||||
|
||||
|
||||
|
||||
@@ -8,9 +8,6 @@ import sys
|
||||
import tempfile
|
||||
import urllib
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
eggs.require( "numpy" )
|
||||
import numpy
|
||||
from bx.intervals.io import GenomicIntervalReader, ParseError
|
||||
|
||||
|
||||
@@ -14,8 +14,6 @@ import weakref
|
||||
|
||||
from os.path import abspath
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "SQLAlchemy >= 0.4" )
|
||||
from sqlalchemy.orm import object_session
|
||||
|
||||
import galaxy.model
|
||||
|
||||
@@ -10,7 +10,7 @@ import re
|
||||
import string
|
||||
from cgi import escape
|
||||
|
||||
from galaxy import eggs, util
|
||||
from galaxy import util
|
||||
from galaxy.datatypes import metadata
|
||||
from galaxy.datatypes.checkers import is_gzip
|
||||
from galaxy.datatypes.metadata import MetadataElement
|
||||
@@ -19,11 +19,7 @@ from galaxy.datatypes.util.image_util import check_image_type
|
||||
from galaxy.util import nice_size
|
||||
from . import data
|
||||
|
||||
try:
|
||||
eggs.require( "bx-python" )
|
||||
import bx.align.maf
|
||||
except:
|
||||
pass
|
||||
import bx.align.maf
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -3,8 +3,6 @@ Provides utilities for working with GFF files.
|
||||
"""
|
||||
import copy
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "bx-python" )
|
||||
from bx.intervals.io import GenomicInterval, MissingFieldError, NiceReaderWrapper, ParseError, GenomicIntervalReader
|
||||
from bx.tabular.io import Header, Comment
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
Determine what optional dependencies are needed.
|
||||
"""
|
||||
import pkg_resources
|
||||
|
||||
from os.path import dirname, join
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from galaxy.util import asbool
|
||||
from galaxy.util.properties import load_app_properties
|
||||
|
||||
|
||||
class ConditionalDependencies( object ):
|
||||
def __init__( self, config_file ):
|
||||
self.config_file = config_file
|
||||
self.config = None
|
||||
self.job_runners = []
|
||||
self.conditional_reqs = []
|
||||
self.parse_configs()
|
||||
self.get_conditional_requirements()
|
||||
|
||||
def parse_configs( self ):
|
||||
self.config = load_app_properties( ini_file=self.config_file )
|
||||
job_conf_xml = self.config.get(
|
||||
"job_config_file",
|
||||
join( dirname( self.config_file ), 'job_conf.xml' ) )
|
||||
try:
|
||||
for plugin in ElementTree.parse( job_conf_xml ).find( 'plugins' ):
|
||||
self.job_runners.append( plugin.attrib['load'] )
|
||||
except (OSError, IOError):
|
||||
pass
|
||||
|
||||
def get_conditional_requirements( self ):
|
||||
crfile = join( dirname( __file__ ), 'conditional-requirements.txt' )
|
||||
for req in pkg_resources.parse_requirements( open( crfile ).readlines() ):
|
||||
self.conditional_reqs.append( req )
|
||||
|
||||
def check( self, name ):
|
||||
try:
|
||||
name = name.replace('-', '_').replace('.', '_')
|
||||
return getattr( self, 'check_' + name )()
|
||||
except:
|
||||
return False
|
||||
|
||||
def check_psycopg2( self ):
|
||||
return self.config["database_connection"].startswith( "postgres" )
|
||||
|
||||
def check_mysql_python( self ):
|
||||
return self.config["database_connection"].startswith( "mysql" )
|
||||
|
||||
def check_drmaa( self ):
|
||||
return ("galaxy.jobs.runners.drmaa:DRMAAJobRunner" in self.job_runners or
|
||||
"galaxy.jobs.runners.slurm:SlurmJobRunner" in self.job_runners)
|
||||
|
||||
def check_pbs_python( self ):
|
||||
return "galaxy.jobs.runners.pbs:PBSJobRunner" in self.job_runners
|
||||
|
||||
def check_python_openid( self ):
|
||||
return asbool( self.config["enable_openid"] )
|
||||
|
||||
def check_fluent_logger( self ):
|
||||
return asbool( self.config["fluent_log"] )
|
||||
|
||||
def check_raven( self ):
|
||||
return asbool( self.config["sentry_dsn"] )
|
||||
|
||||
def check_weberror( self ):
|
||||
return ( asbool( self.config["debug"] ) and
|
||||
asbool( self.config["use_interactive"] ) )
|
||||
|
||||
|
||||
def optional( config_file ):
|
||||
rval = []
|
||||
conditional = ConditionalDependencies( config_file )
|
||||
for opt in conditional.conditional_reqs:
|
||||
if conditional.check( opt.key ):
|
||||
rval.append( str( opt ) )
|
||||
return rval
|
||||
@@ -0,0 +1,10 @@
|
||||
# These dependencies are only required when certain config options are set
|
||||
psycopg2==2.6.1
|
||||
WebError==0.11
|
||||
python-openid
|
||||
MySQL-python
|
||||
fluent-logger
|
||||
raven
|
||||
pbs_python
|
||||
drmaa
|
||||
# PyRods not in PyPI
|
||||
@@ -0,0 +1,3 @@
|
||||
nose
|
||||
NoseHTML
|
||||
twill==0.9.1
|
||||
@@ -0,0 +1,64 @@
|
||||
# packages with C extensions
|
||||
Cheetah==2.4.4
|
||||
bx-python==0.7.3
|
||||
MarkupSafe==0.23
|
||||
PyYAML==3.11
|
||||
SQLAlchemy==1.0.8
|
||||
mercurial==3.5
|
||||
numpy==1.9.2
|
||||
pycrypto==2.6.1
|
||||
python_lzo==1.8
|
||||
|
||||
# extra functionality from Dannon
|
||||
pysam==0.8.3+gx1
|
||||
|
||||
# pure Python packages
|
||||
Paste==1.7.5.1
|
||||
PasteDeploy==1.5.2
|
||||
docutils==0.12
|
||||
wchartype==0.1
|
||||
repoze.lru==0.6
|
||||
Routes==2.2
|
||||
WebOb==1.4.1
|
||||
WebHelpers==1.3
|
||||
Mako==1.0.2
|
||||
pytz==2015.4
|
||||
Babel==2.0
|
||||
Beaker==1.7.0
|
||||
|
||||
# BioBlend and dependencies
|
||||
bioblend==0.6.1
|
||||
boto==2.38.0
|
||||
requests==2.8.1
|
||||
|
||||
# WebError and dependences
|
||||
WebError==0.11
|
||||
Pygments==2.0.2
|
||||
|
||||
# kombu and dependencies
|
||||
kombu==3.0.26
|
||||
amqp==1.4.6
|
||||
anyjson==0.3.3
|
||||
|
||||
# sqlalchemy-migrate and dependencies
|
||||
sqlalchemy-migrate==0.10.0
|
||||
decorator==4.0.2
|
||||
Tempita==0.5.3dev
|
||||
sqlparse==0.1.16
|
||||
pbr==1.8.0
|
||||
six==1.9.0 # six is also a Pulsar client dep
|
||||
Parsley==1.3
|
||||
nose==1.3.7
|
||||
SVGFig==1.1.6
|
||||
|
||||
# Fabric and dependencies
|
||||
Fabric==1.10.2
|
||||
paramiko==1.15.2
|
||||
ecdsa==0.13
|
||||
# pycrypto, but it is a direct Galaxy dependency as well
|
||||
|
||||
# Can't use Whoosh latest due to a bug but need to backport a bugfix from a
|
||||
# newer release.
|
||||
#
|
||||
# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff
|
||||
Whoosh==2.4.1+gx1
|
||||
+2
-532
@@ -1,538 +1,8 @@
|
||||
"""
|
||||
Manage Galaxy eggs
|
||||
For backwards compatibility
|
||||
"""
|
||||
|
||||
import ConfigParser
|
||||
import glob
|
||||
import HTMLParser
|
||||
import os
|
||||
import pkg_resources
|
||||
import shutil
|
||||
import sys
|
||||
import urllib
|
||||
import urllib2
|
||||
import zipfile
|
||||
import zipimport
|
||||
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
log.addHandler( logging.NullHandler() )
|
||||
|
||||
|
||||
galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), os.pardir, os.pardir, os.pardir ) )
|
||||
eggs_dir = os.environ.get( 'GALAXY_EGGS_PATH', os.path.join( galaxy_dir, 'eggs' ) )
|
||||
py = 'py%s' % sys.version[:3]
|
||||
|
||||
|
||||
class EggNotFetchable( Exception ):
|
||||
def __init__( self, eggs ):
|
||||
if type( eggs ) in ( list, tuple ):
|
||||
self.eggs = eggs
|
||||
else:
|
||||
self.eggs = [ eggs ]
|
||||
|
||||
def __str__( self ):
|
||||
return ' '.join( self.eggs )
|
||||
|
||||
|
||||
# need the options to remain case sensitive
|
||||
class CaseSensitiveConfigParser( ConfigParser.SafeConfigParser ):
|
||||
def optionxform( self, optionstr ):
|
||||
return optionstr
|
||||
|
||||
|
||||
# so we can actually detect failures
|
||||
class URLRetriever( urllib.FancyURLopener ):
|
||||
def http_error_default( *args ):
|
||||
urllib.URLopener.http_error_default( *args )
|
||||
|
||||
|
||||
class Egg( object ):
|
||||
"""
|
||||
Contains information about locating and downloading eggs.
|
||||
"""
|
||||
def __init__( self, name=None, version=None, tag=None, url=None, platform=None, crate=None ):
|
||||
self.name = name
|
||||
self.version = version
|
||||
self.tag = tag
|
||||
self.url = url
|
||||
self.platform = platform
|
||||
self.crate = crate
|
||||
self.distribution = None
|
||||
self.dir = None
|
||||
self.removed_location = None
|
||||
self.enable_fetch = crate.enable_egg_fetch
|
||||
if self.name is not None and self.version is not None:
|
||||
self.set_distribution()
|
||||
|
||||
def set_dir( self ):
|
||||
global eggs_dir
|
||||
self.dir = eggs_dir
|
||||
|
||||
def set_distribution( self ):
|
||||
"""
|
||||
Stores a pkg_resources Distribution object for reference later
|
||||
"""
|
||||
if self.dir is None:
|
||||
self.set_dir()
|
||||
tag = self.tag or ''
|
||||
self.distribution = pkg_resources.Distribution.from_filename(
|
||||
os.path.join( self.dir, '-'.join( ( self.name, self.version + tag, self.platform ) ) + '.egg' ) )
|
||||
|
||||
@property
|
||||
def path( self ):
|
||||
"""
|
||||
Return the path of the egg, if it exists, or None
|
||||
"""
|
||||
if env[self.distribution.project_name]:
|
||||
return env[self.distribution.project_name][0].location
|
||||
return None
|
||||
|
||||
def fetch( self, requirement ):
|
||||
"""
|
||||
fetch() serves as the install method to pkg_resources.working_set.resolve()
|
||||
"""
|
||||
def find_alternative():
|
||||
"""
|
||||
Some platforms (e.g. Solaris) support eggs compiled on older platforms
|
||||
"""
|
||||
class LinkParser( HTMLParser.HTMLParser ):
|
||||
"""
|
||||
Finds links in what should be an Apache-style directory index
|
||||
"""
|
||||
def __init__( self ):
|
||||
HTMLParser.HTMLParser.__init__( self )
|
||||
self.links = []
|
||||
|
||||
def handle_starttag( self, tag, attrs ):
|
||||
if tag == 'a' and 'href' in dict( attrs ):
|
||||
self.links.append( dict( attrs )['href'] )
|
||||
|
||||
parser = LinkParser()
|
||||
try:
|
||||
parser.feed( urllib2.urlopen( self.url + '/' ).read() )
|
||||
except urllib2.HTTPError, e:
|
||||
if e.code == 404:
|
||||
return None
|
||||
parser.close()
|
||||
for link in parser.links:
|
||||
file = urllib.unquote( link ).rsplit( '/', 1 )[-1]
|
||||
tmp_dist = pkg_resources.Distribution.from_filename( file )
|
||||
if tmp_dist.platform is not None and \
|
||||
self.distribution.project_name == tmp_dist.project_name and \
|
||||
self.distribution.version == tmp_dist.version and \
|
||||
self.distribution.py_version == tmp_dist.py_version and \
|
||||
pkg_resources.compatible_platforms( tmp_dist.platform, pkg_resources.get_platform() ):
|
||||
return file
|
||||
return None
|
||||
|
||||
def _fetch():
|
||||
if self.url is None:
|
||||
return False
|
||||
if not os.path.exists( self.dir ):
|
||||
os.makedirs( self.dir )
|
||||
alternative = None
|
||||
try:
|
||||
url = self.url + '/' + self.distribution.egg_name() + '.egg'
|
||||
URLRetriever().retrieve( url, self.distribution.location )
|
||||
log.debug( "Fetched %s" % url )
|
||||
except IOError, e:
|
||||
if e[1] == 404 and self.distribution.platform != py:
|
||||
alternative = find_alternative()
|
||||
if alternative is None:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
if alternative is not None:
|
||||
try:
|
||||
url = '/'.join( ( self.url, alternative ) )
|
||||
URLRetriever().retrieve( url, os.path.join( self.dir, alternative ) )
|
||||
log.debug( "Fetched %s" % url )
|
||||
except IOError, e:
|
||||
return False
|
||||
self.platform = alternative.split( '-', 2 )[-1].rsplit( '.egg', 1 )[0]
|
||||
self.set_distribution()
|
||||
self.unpack_if_needed()
|
||||
self.remove_doppelgangers()
|
||||
return True
|
||||
# If being called from a version conflict, that code has removed a
|
||||
# directory from sys.path. That directory could be Python's main lib
|
||||
# directory (e.g. in the case of wsgiref under 2.6). Temporarily put
|
||||
# it back during the execution of this code.
|
||||
if self.removed_location:
|
||||
sys.path.append( self.removed_location )
|
||||
try:
|
||||
assert _fetch()
|
||||
rval = self.distribution
|
||||
except:
|
||||
rval = None
|
||||
if self.removed_location:
|
||||
sys.path.remove( self.removed_location )
|
||||
if rval is not None:
|
||||
global env
|
||||
env = get_env() # reset the global Environment object now that we've obtained a new egg
|
||||
return rval
|
||||
|
||||
def unpack_if_needed( self ):
|
||||
meta = pkg_resources.EggMetadata( zipimport.zipimporter( self.distribution.location ) )
|
||||
if meta.has_metadata( 'not-zip-safe' ):
|
||||
unpack_zipfile( self.distribution.location, self.distribution.location + "-tmp" )
|
||||
os.remove( self.distribution.location )
|
||||
os.rename( self.distribution.location + "-tmp", self.distribution.location )
|
||||
|
||||
def remove_doppelgangers( self ):
|
||||
doppelgangers = glob.glob( os.path.join( self.dir, "%s-*-%s.egg" % ( self.name, self.platform ) ) )
|
||||
if self.distribution.location in doppelgangers:
|
||||
doppelgangers.remove( self.distribution.location )
|
||||
for doppelganger in doppelgangers:
|
||||
remove_file_or_path( doppelganger )
|
||||
log.debug( "Removed conflicting egg: %s" % doppelganger )
|
||||
|
||||
def resolve( self ):
|
||||
try:
|
||||
rval = []
|
||||
# resolve this egg and its dependencies
|
||||
if self.enable_fetch:
|
||||
dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, self.fetch )
|
||||
else:
|
||||
dists = pkg_resources.working_set.resolve( ( self.distribution.as_requirement(), ), env, lambda x: None )
|
||||
for dist in dists:
|
||||
# if any of the resolved dependencies should be managed eggs but are being pulled from the wrong path, fix them
|
||||
if dist.project_name in self.crate.all_names and not os.path.realpath( dist.location ).startswith( os.path.realpath( self.dir ) ):
|
||||
# TODO: like eggs.require(), this breaks requirement versioning
|
||||
subdists = self.version_conflict( dist, dist.as_requirement() )
|
||||
if type( subdists ) == list:
|
||||
rval.extend( subdists )
|
||||
else:
|
||||
rval.append( subdists )
|
||||
else:
|
||||
rval.append( dist )
|
||||
return rval
|
||||
except pkg_resources.DistributionNotFound, e:
|
||||
# If this statement is true, it means we do have the requested egg,
|
||||
# just not one (or more) of its deps.
|
||||
if e.args[0].project_name != self.distribution.project_name:
|
||||
log.warning( "Warning: %s (a dependent egg of %s) cannot be fetched" % ( e.args[0].project_name, self.distribution.project_name ) )
|
||||
return ( self.distribution, )
|
||||
else:
|
||||
raise EggNotFetchable( self )
|
||||
except pkg_resources.VersionConflict, e:
|
||||
if e.args[1].key != e.args[0].key:
|
||||
log.error( "One of Galaxy's managed eggs depends on something which is missing, this is almost certainly a bug in the egg distribution." )
|
||||
log.error( 'Dependency "%s" requires "%s"' % ( e.args[0].project_name, str( e.args[1] ) ) )
|
||||
raise
|
||||
# there's a conflicting egg on the path, remove it
|
||||
return self.version_conflict( e.args[0], e.args[1] )
|
||||
|
||||
def version_conflict( self, conflict_dist, conflict_req ):
|
||||
# since this conflict may be for a dependent egg, find the correct egg from the crate
|
||||
if conflict_dist.project_name == self.distribution.project_name:
|
||||
egg = self
|
||||
dist = egg.distribution
|
||||
elif conflict_dist.project_name in self.crate.all_names:
|
||||
egg = self.crate[conflict_dist.project_name]
|
||||
dist = egg.distribution
|
||||
else:
|
||||
# should not happen, but just in case
|
||||
egg = None
|
||||
dist = conflict_dist
|
||||
# use the canonical path to locate and remove the conflict from the working set
|
||||
location = os.path.realpath( conflict_dist.location )
|
||||
for entry in pkg_resources.working_set.entries:
|
||||
if os.path.realpath( entry ) == location:
|
||||
pkg_resources.working_set.entries.remove( entry )
|
||||
break
|
||||
else:
|
||||
location = entry = None
|
||||
del pkg_resources.working_set.by_key[conflict_dist.key]
|
||||
# remove the conflict from sys.path
|
||||
if entry is not None:
|
||||
pkg_resources.working_set.entry_keys[entry] = []
|
||||
if entry in sys.path:
|
||||
sys.path.remove(entry)
|
||||
# if the conflict is a dependent egg, fetch that specific egg
|
||||
if egg:
|
||||
# Store the removed path so the fetch method can use it
|
||||
egg.removed_location = location
|
||||
if self.enable_fetch:
|
||||
r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, egg.fetch )
|
||||
else:
|
||||
r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env, lambda x: None )
|
||||
egg.removed_location = None
|
||||
else:
|
||||
r = pkg_resources.working_set.resolve( ( dist.as_requirement(), ), env )
|
||||
# re-add the path if it's a non-egg dir, in case more deps live there
|
||||
if location is not None and not location.endswith( '.egg' ):
|
||||
pkg_resources.working_set.entries.append( location )
|
||||
sys.path.append( location )
|
||||
return r
|
||||
|
||||
def require( self ):
|
||||
try:
|
||||
dists = self.resolve()
|
||||
for dist in dists:
|
||||
if dist.location not in pkg_resources.working_set.entries:
|
||||
pkg_resources.working_set.add( dist )
|
||||
return dists
|
||||
except:
|
||||
raise
|
||||
|
||||
|
||||
class Crate( object ):
|
||||
"""
|
||||
Reads the eggs.ini file for use with checking and fetching.
|
||||
"""
|
||||
config_file = os.path.join( galaxy_dir, 'eggs.ini' )
|
||||
|
||||
def __init__( self, galaxy_config_file=None, platform=None ):
|
||||
self.eggs = {}
|
||||
self.config = CaseSensitiveConfigParser()
|
||||
self.repo = None
|
||||
self.no_auto = []
|
||||
self.platform = platform
|
||||
self.py_platform = None
|
||||
if platform is not None:
|
||||
self.py_platform = platform.split( '-' )[0]
|
||||
self.enable_egg_fetch = string_as_bool(os.environ.get('GALAXY_ENABLE_EGG_FETCH', True))
|
||||
self.enable_eggs = string_as_bool(os.environ.get('GALAXY_ENABLE_EGGS', True))
|
||||
self.try_dependencies_from_env = string_as_bool(os.environ.get('GALAXY_TRY_DEPENDENCIES_FROM_ENV', False))
|
||||
self.galaxy_config = GalaxyConfig( galaxy_config_file )
|
||||
self.parse()
|
||||
|
||||
def parse( self ):
|
||||
self.config.read( Crate.config_file )
|
||||
self.repo = self.config.get( 'general', 'repository' )
|
||||
self.no_auto = self.config.get( 'general', 'no_auto' ).split()
|
||||
self.parse_egg_section( self.config.items( 'eggs:platform' ), self.config.items( 'tags' ), True )
|
||||
self.parse_egg_section( self.config.items( 'eggs:noplatform' ), self.config.items( 'tags' ) )
|
||||
|
||||
def parse_egg_section( self, eggs, tags, full_platform=False, egg_class=Egg ):
|
||||
for name, version in eggs:
|
||||
tag = dict( tags ).get( name, '' )
|
||||
url = '/'.join( ( self.repo, name ) )
|
||||
if full_platform:
|
||||
platform = self.platform or '-'.join( ( py, pkg_resources.get_platform() ) )
|
||||
else:
|
||||
platform = self.py_platform or py
|
||||
egg = egg_class( name=name, version=version, tag=tag, url=url, platform=platform, crate=self )
|
||||
self.eggs[name] = egg
|
||||
|
||||
@property
|
||||
def config_missing( self ):
|
||||
"""
|
||||
Return true if any eggs are missing, conditional on options set in the
|
||||
Galaxy config file.
|
||||
"""
|
||||
for egg in self.config_eggs:
|
||||
if not egg.path:
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def all_missing( self ):
|
||||
"""
|
||||
Return true if any eggs in the eggs config file are missing.
|
||||
"""
|
||||
for egg in self.all_eggs:
|
||||
if not os.path.exists( egg.distribution.location ):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def config_names( self ):
|
||||
"""
|
||||
Return a list of names of all eggs in the crate that are needed based
|
||||
on the options set in the Galaxy config file.
|
||||
"""
|
||||
return [ egg.name for egg in self.config_eggs ]
|
||||
|
||||
@property
|
||||
def all_names( self ):
|
||||
"""
|
||||
Return a list of names of all eggs in the crate.
|
||||
"""
|
||||
return [ egg.name for egg in self.all_eggs ]
|
||||
|
||||
@property
|
||||
def config_eggs( self ):
|
||||
"""
|
||||
Return a list of all eggs in the crate that are needed based on the
|
||||
options set in the Galaxy config file.
|
||||
"""
|
||||
return [ egg for egg in self.eggs.values() if self.galaxy_config.check_conditional( egg.name ) ]
|
||||
|
||||
@property
|
||||
def all_eggs( self ):
|
||||
"""
|
||||
Return a list of all eggs in the crate.
|
||||
"""
|
||||
rval = []
|
||||
for egg in self.eggs.values():
|
||||
if egg.name in self.galaxy_config.always_conditional and not self.galaxy_config.check_conditional( egg.name ):
|
||||
continue
|
||||
rval.append( egg )
|
||||
return rval
|
||||
|
||||
def __getitem__( self, name ):
|
||||
"""
|
||||
Return a specific egg.
|
||||
"""
|
||||
name = name.replace( '-', '_' )
|
||||
return self.eggs[name]
|
||||
|
||||
def resolve( self, all=False ):
|
||||
"""
|
||||
Try to resolve (e.g. fetch) all eggs in the crate.
|
||||
"""
|
||||
if self.enable_eggs and not self.try_dependencies_from_env:
|
||||
if all:
|
||||
eggs = self.all_eggs
|
||||
else:
|
||||
eggs = self.config_eggs
|
||||
eggs = filter( lambda x: x.name not in self.no_auto, eggs )
|
||||
missing = []
|
||||
for egg in eggs:
|
||||
try:
|
||||
egg.resolve()
|
||||
except EggNotFetchable:
|
||||
missing.append( egg )
|
||||
if missing:
|
||||
raise EggNotFetchable( missing )
|
||||
else:
|
||||
log.info('Dependencies will attempt to be loaded from the environment')
|
||||
|
||||
|
||||
class GalaxyConfig( object ):
|
||||
always_conditional = ( 'pysam', 'ctypes', 'python_daemon' )
|
||||
|
||||
def __init__( self, config_file ):
|
||||
if config_file is None:
|
||||
self.config = None
|
||||
else:
|
||||
self.config = ConfigParser.ConfigParser()
|
||||
if self.config.read( config_file ) == []:
|
||||
raise Exception( "error: unable to read Galaxy config from %s" % config_file )
|
||||
|
||||
def check_conditional( self, egg_name ):
|
||||
def check_pysam():
|
||||
# can't build pysam on solaris < 10
|
||||
plat = pkg_resources.get_platform().split( '-' )
|
||||
if plat[0] == 'solaris':
|
||||
minor = plat[1].split('.')[1]
|
||||
if int( minor ) < 10:
|
||||
return False
|
||||
return True
|
||||
# If we're using require() we may not have a Galaxy config file, but if
|
||||
# we're using require(), we don't care about conditionals.
|
||||
if self.config is None:
|
||||
return True
|
||||
if egg_name == "pysqlite":
|
||||
# SQLite is different since it can be specified in two config vars and defaults to True
|
||||
try:
|
||||
return self.config.get( "app:main", "database_connection" ).startswith( "sqlite://" )
|
||||
except:
|
||||
return True
|
||||
else:
|
||||
try:
|
||||
return { "psycopg2":
|
||||
lambda: self.config.get( "app:main", "database_connection" ).startswith( "postgres" ),
|
||||
"MySQL_python":
|
||||
lambda: self.config.get( "app:main", "database_connection" ).startswith( "mysql://" ),
|
||||
"DRMAA_python":
|
||||
lambda: "sge" in self.config.get( "app:main", "start_job_runners" ).split(","),
|
||||
"drmaa":
|
||||
lambda: "drmaa" in self.config.get( "app:main", "start_job_runners" ).split(","),
|
||||
"pbs_python":
|
||||
lambda: "pbs" in self.config.get( "app:main", "start_job_runners" ).split(","),
|
||||
"python_openid":
|
||||
lambda: self.config.get( "app:main", "enable_openid" ),
|
||||
"python_daemon":
|
||||
lambda: sys.version_info[:2] >= ( 2, 5 ),
|
||||
"pysam":
|
||||
lambda: check_pysam(),
|
||||
"PyRods":
|
||||
lambda: self.config.get( "app:main", "object_store" ) == "irods"
|
||||
}.get( egg_name, lambda: True )()
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
def string_as_bool( string ):
|
||||
if str( string ).lower() in ( 'true', 'yes', 'on' ):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def get_env():
|
||||
env = pkg_resources.Environment( search_path='', platform=pkg_resources.get_platform() )
|
||||
for dist in pkg_resources.find_distributions( eggs_dir, False ):
|
||||
env.add( dist )
|
||||
return env
|
||||
env = get_env()
|
||||
|
||||
|
||||
def require( req_str ):
|
||||
c = Crate( None )
|
||||
req = pkg_resources.Requirement.parse( req_str )
|
||||
if c.try_dependencies_from_env or not c.enable_eggs:
|
||||
try:
|
||||
return pkg_resources.working_set.require( req_str )
|
||||
except Exception as exc:
|
||||
if not c.enable_eggs:
|
||||
raise
|
||||
log.info("%s not found in local environment, will try Galaxy egg: %s", (req_str, exc))
|
||||
# TODO: This breaks egg version requirements. Not currently a problem, but
|
||||
# it could become one.
|
||||
try:
|
||||
return c[req.project_name].require()
|
||||
except KeyError:
|
||||
# not a galaxy-owned dependency
|
||||
return pkg_resources.working_set.require( req_str )
|
||||
except EggNotFetchable, e:
|
||||
raise EggNotFetchable( str( [ egg.name for egg in e.eggs ] ) )
|
||||
pkg_resources.require = require
|
||||
|
||||
|
||||
def unpack_zipfile( filename, extract_dir, ignores=[] ):
|
||||
z = zipfile.ZipFile(filename)
|
||||
try:
|
||||
for info in z.infolist():
|
||||
name = info.filename
|
||||
mode = (info.external_attr >> 16L) & 0777
|
||||
# don't extract absolute paths or ones with .. in them
|
||||
if name.startswith('/') or '..' in name:
|
||||
continue
|
||||
target = os.path.join(extract_dir, *name.split('/'))
|
||||
if not target:
|
||||
continue
|
||||
for ignore in ignores:
|
||||
if ignore in name:
|
||||
continue
|
||||
if name.endswith('/'):
|
||||
# directory
|
||||
pkg_resources.ensure_directory(target)
|
||||
else:
|
||||
# file
|
||||
pkg_resources.ensure_directory(target)
|
||||
data = z.read(info.filename)
|
||||
f = open(target, 'wb')
|
||||
try:
|
||||
f.write(data)
|
||||
finally:
|
||||
f.close()
|
||||
del data
|
||||
try:
|
||||
if not os.path.islink():
|
||||
os.chmod(target, mode)
|
||||
except:
|
||||
pass
|
||||
finally:
|
||||
z.close()
|
||||
|
||||
|
||||
def remove_file_or_path( f ):
|
||||
if os.path.isdir( f ):
|
||||
shutil.rmtree( f )
|
||||
else:
|
||||
os.remove( f )
|
||||
require = pkg_resources.require
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Manage Galaxy eggs
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from scramble import ScrambleEgg, ScrambleCrate, ScrambleFailure, galaxy_dir
|
||||
from __init__ import Crate, CaseSensitiveConfigParser
|
||||
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
log.addHandler( logging.NullHandler() )
|
||||
|
||||
|
||||
class DistScrambleEgg( ScrambleEgg ):
|
||||
def set_dir( self ):
|
||||
self.dir = os.path.join( galaxy_dir, 'dist-eggs', self.name )
|
||||
if not os.path.exists( self.dir ):
|
||||
os.makedirs( self.dir )
|
||||
|
||||
@property
|
||||
def path( self ):
|
||||
# don't look for compatible eggs, look for exact matches
|
||||
if os.path.exists( self.distribution.location ):
|
||||
return self.distribution.location
|
||||
return None
|
||||
|
||||
def run_scramble_script( self ):
|
||||
log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name )
|
||||
# subprocessed to sterilize the env
|
||||
cmd = "ssh %s 'cd %s; PYTHONPATH= %s -ES %s'" % ( self.build_host, self.buildpath, self.python, 'scramble.py' )
|
||||
log.debug( '%s(): Executing:' % sys._getframe().f_code.co_name )
|
||||
log.debug( ' %s' % cmd )
|
||||
p = subprocess.Popen( args=cmd, shell=True )
|
||||
r = p.wait()
|
||||
if r != 0:
|
||||
raise ScrambleFailure( "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) )
|
||||
|
||||
def unpack_if_needed( self ):
|
||||
return # do not unpack dist eggs
|
||||
|
||||
|
||||
class DistScrambleCrate( ScrambleCrate ):
|
||||
"""
|
||||
Holds eggs with info on how to build them for distribution.
|
||||
"""
|
||||
dist_config_file = os.path.join( galaxy_dir, 'dist-eggs.ini' )
|
||||
|
||||
def __init__( self, galaxy_config_file, build_on='all' ):
|
||||
self.dist_config = CaseSensitiveConfigParser()
|
||||
self.build_on = build_on
|
||||
ScrambleCrate.__init__( self, galaxy_config_file )
|
||||
|
||||
def parse( self ):
|
||||
self.dist_config.read( DistScrambleCrate.dist_config_file )
|
||||
self.hosts = dict( self.dist_config.items( 'hosts' ) )
|
||||
self.groups = dict( self.dist_config.items( 'groups' ) )
|
||||
self.ignore = dict( self.dist_config.items( 'ignore' ) )
|
||||
self.platforms = self.get_platforms( self.build_on )
|
||||
self.noplatforms = self.get_platforms( 'noplatform' )
|
||||
Crate.parse( self )
|
||||
|
||||
def get_platforms( self, wanted ):
|
||||
# find all the members of a group and process them
|
||||
if wanted in self.groups:
|
||||
platforms = []
|
||||
for name in self.groups[wanted].split():
|
||||
for platform in self.get_platforms( name ):
|
||||
if platform not in platforms:
|
||||
platforms.append( platform )
|
||||
return platforms
|
||||
elif wanted in self.hosts:
|
||||
return [ wanted ]
|
||||
else:
|
||||
raise Exception( "unknown platform: %s" % wanted )
|
||||
|
||||
def parse_egg_section( self, eggs, tags, full_platform=False ):
|
||||
for name, version in eggs:
|
||||
self.eggs[name] = []
|
||||
tag = dict( tags ).get( name, '' )
|
||||
url = '/'.join( ( self.repo, name ) )
|
||||
try:
|
||||
sources = self.config.get( 'source', name ).split()
|
||||
except:
|
||||
sources = []
|
||||
try:
|
||||
dependencies = self.config.get( 'dependencies', name ).split()
|
||||
except:
|
||||
dependencies = []
|
||||
if full_platform:
|
||||
platforms = self.platforms
|
||||
else:
|
||||
platforms = self.noplatforms
|
||||
for platform in platforms:
|
||||
if name in self.ignore and platform in self.ignore[name].split():
|
||||
continue
|
||||
egg = DistScrambleEgg( name, version, tag, url, platform, self )
|
||||
host_info = self.hosts[platform].split()
|
||||
egg.build_host, egg.python = host_info[:2]
|
||||
egg.sources = sources
|
||||
egg.dependencies = dependencies
|
||||
self.eggs[name].append( egg )
|
||||
@@ -1,268 +0,0 @@
|
||||
"""
|
||||
Manage Galaxy eggs
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import subprocess
|
||||
import urlparse
|
||||
import urllib
|
||||
from __init__ import Egg, Crate, URLRetriever, galaxy_dir, py, unpack_zipfile
|
||||
from distutils.sysconfig import get_config_var
|
||||
|
||||
import tarfile
|
||||
import zipfile
|
||||
arctypes = ( 'tar.gz', 'tgz', 'zip' )
|
||||
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
log.addHandler( logging.NullHandler() )
|
||||
|
||||
|
||||
class ScrambleFailure( Exception ):
|
||||
def __init__( self, eggs, msg=None ):
|
||||
if type( eggs ) in ( list, tuple ):
|
||||
self.eggs = eggs
|
||||
else:
|
||||
self.eggs = [ eggs ]
|
||||
self.msg = msg
|
||||
|
||||
def __str__( self ):
|
||||
return self.msg or ' '.join( self.eggs )
|
||||
|
||||
|
||||
class ScrambleEgg( Egg ):
|
||||
"""
|
||||
Contains information about scrambling eggs.
|
||||
"""
|
||||
scramble_dir = os.path.join( galaxy_dir, 'scripts', 'scramble' )
|
||||
archive_dir = os.path.join( scramble_dir, 'archives' )
|
||||
script_dir = os.path.join( scramble_dir, 'scripts' )
|
||||
build_dir = os.path.join( scramble_dir, 'build' )
|
||||
ez_setup = os.path.join( scramble_dir, 'lib', 'ez_setup.py' )
|
||||
ez_setup_url = 'http://peak.telecommunity.com/dist/ez_setup.py'
|
||||
|
||||
def __init__( self, *args, **kwargs ):
|
||||
Egg.__init__( self, *args, **kwargs )
|
||||
self.sources = []
|
||||
self.dependencies = []
|
||||
self.buildpath = None
|
||||
self.source_path = None
|
||||
self.py = py
|
||||
self.build_host = None
|
||||
self.python = sys.executable
|
||||
|
||||
def scramble( self ):
|
||||
if self.path:
|
||||
log.warning( "%s(): Egg already exists, remove to force rebuild:" % sys._getframe().f_code.co_name )
|
||||
log.warning( " %s" % self.path )
|
||||
return
|
||||
self.fetch_source()
|
||||
self.unpack_source()
|
||||
self.copy_build_script()
|
||||
if not os.path.exists( ScrambleEgg.ez_setup ):
|
||||
URLRetriever().retrieve( ScrambleEgg.ez_setup_url, ScrambleEgg.ez_setup )
|
||||
self.run_scramble_script()
|
||||
new_egg = os.path.join( self.buildpath, 'dist', os.path.basename( self.distribution.location ) )
|
||||
if not os.path.exists( new_egg ):
|
||||
raise ScrambleFailure( self, "%s(): Egg build for %s did not appear to fail, but no egg found to copy from expected path:\n %s" % ( sys._getframe().f_code.co_name, self.name, new_egg ) )
|
||||
shutil.copyfile( new_egg, self.distribution.location )
|
||||
log.warning( "%s(): Copied egg to:" % sys._getframe().f_code.co_name )
|
||||
log.warning( " %s" % self.distribution.location )
|
||||
self.unpack_if_needed()
|
||||
self.remove_doppelgangers()
|
||||
|
||||
# scramble helper methods
|
||||
def get_tld( self, names ):
|
||||
tld = names[0].split( os.path.sep, 1 )[0]
|
||||
for name in names:
|
||||
try:
|
||||
assert tld == name.split( os.path.sep, 1 )[0]
|
||||
except:
|
||||
raise Exception( "%s(): Archive contains multiple top-level directories!" % sys._getframe().f_code.co_name )
|
||||
return tld
|
||||
|
||||
def fetch_one( self, urls ):
|
||||
"""
|
||||
Fetches the first available archive out of a list.
|
||||
"""
|
||||
for url in urls:
|
||||
file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] )
|
||||
if os.path.exists( file ):
|
||||
log.warning( "%s(): Using existing source, remove to download again:" % sys._getframe().f_code.co_name )
|
||||
log.warning( " %s" % file )
|
||||
return file
|
||||
# if we don't have one, get one
|
||||
for url in urls:
|
||||
file = os.path.join( ScrambleEgg.archive_dir, ( urllib.unquote( url ).rsplit( '/', 1 ) )[-1] )
|
||||
try:
|
||||
log.debug( "%s(): Trying to fetch:" % sys._getframe().f_code.co_name )
|
||||
log.debug( " %s" % url )
|
||||
URLRetriever().retrieve( url, file + '.download' )
|
||||
shutil.move( file + '.download', file )
|
||||
log.debug( "%s(): Fetched to:" % sys._getframe().f_code.co_name )
|
||||
log.debug( " %s" % file )
|
||||
return file
|
||||
except IOError, e:
|
||||
if e[1] != 404:
|
||||
raise
|
||||
else:
|
||||
return None
|
||||
|
||||
def fetch_source( self ):
|
||||
"""
|
||||
Get egg (and dependent) source
|
||||
"""
|
||||
if not os.path.exists( ScrambleEgg.archive_dir ):
|
||||
os.makedirs( ScrambleEgg.archive_dir )
|
||||
urls = []
|
||||
url_base = self.url + '/' + '-'.join( ( self.name, self.version ) )
|
||||
urls.extend( map( lambda x: '.'.join( ( url_base, x ) ), arctypes ) )
|
||||
if self.tag:
|
||||
urls.extend( map( lambda x: '.'.join( ( url_base + self.tag, x ) ), arctypes ) )
|
||||
self.source_path = self.fetch_one( urls )
|
||||
if self.source_path is None:
|
||||
raise Exception( "%s(): Couldn't find a suitable source archive for %s %s from %s" % ( sys._getframe().f_code.co_name, self.name, self.version, self.url ) )
|
||||
for url in self.sources:
|
||||
if not urlparse.urlparse( url )[0]:
|
||||
url = self.url + '/' + url.lstrip( '/' )
|
||||
urls = [ url ]
|
||||
urls.extend( map( lambda x: '.'.join( ( url, x ) ), arctypes ) ) # allows leaving off the extension and we'll try to find one
|
||||
file = self.fetch_one( urls )
|
||||
if file is None:
|
||||
raise Exception( "%s(): Couldn't fetch extra source for %s, check path in %s. URL(s) attempted output above." % ( sys._getframe().f_code.co_name, self.name, Crate.config_file, ) )
|
||||
|
||||
def unpack_source( self ):
|
||||
unpack_dir = os.path.join( ScrambleEgg.build_dir, self.platform )
|
||||
if not os.path.exists( unpack_dir ):
|
||||
os.makedirs( unpack_dir )
|
||||
self.buildpath = os.path.join( unpack_dir, self.name )
|
||||
if os.path.exists( self.buildpath ):
|
||||
log.warning( "%s(): Removing old build directory at:" % sys._getframe().f_code.co_name )
|
||||
log.warning( " %s" % self.buildpath )
|
||||
shutil.rmtree( self.buildpath )
|
||||
if tarfile.is_tarfile( self.source_path ):
|
||||
self.unpack_tar()
|
||||
elif zipfile.is_zipfile( self.source_path ):
|
||||
self.unpack_zip()
|
||||
else:
|
||||
raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, self.source_path ) )
|
||||
log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name )
|
||||
log.warning( " %s" % self.buildpath )
|
||||
|
||||
def unpack_zip( self ):
|
||||
unpack_path = os.path.dirname( self.buildpath )
|
||||
tld = self.get_tld( zipfile.ZipFile( self.source_path, 'r' ).namelist() )
|
||||
unpack_zipfile( self.source_path, unpack_path, ( 'ez_setup', ) )
|
||||
os.rename( os.path.join( unpack_path, tld ), self.buildpath )
|
||||
|
||||
def unpack_tar( self ):
|
||||
unpack_path = os.path.dirname( self.buildpath )
|
||||
t = tarfile.open( self.source_path, "r" )
|
||||
members = filter( lambda x: "ez_setup" not in x.name and "pax_global_header" != x.name, t.getmembers() )
|
||||
tld = self.get_tld( [ x.name for x in members ] )
|
||||
cur = os.getcwd()
|
||||
os.chdir( unpack_path )
|
||||
for member in members:
|
||||
try:
|
||||
t.extract( member )
|
||||
except:
|
||||
log.exception( "Could not extract %s", member )
|
||||
raise
|
||||
t.close()
|
||||
os.rename( tld, self.name )
|
||||
os.chdir( cur )
|
||||
|
||||
def copy_build_script( self ):
|
||||
# will try:
|
||||
# bx_python-py2.4-solaris-2.11-i86pc.py
|
||||
# bx_python-py2.4-solaris.py
|
||||
# bx_python-solaris-2.11-i86pc.py
|
||||
# bx_python-solaris.py
|
||||
# bx_python-py2.4.py
|
||||
# bx_python.py
|
||||
# generic.py
|
||||
platform = self.platform.replace( '-ucs2', '' ).replace( '-ucs4', '' ) # ucs is unimportant here
|
||||
build_scripts = (
|
||||
"%s-%s.py" % ( self.name, platform ),
|
||||
"%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[:2] ) ),
|
||||
"%s-%s.py" % ( self.name, '-'.join( platform.split( '-' )[1:] ) ),
|
||||
"%s-%s.py" % ( self.name, platform.split( '-' )[:2][-1] ),
|
||||
"%s-%s.py" % ( self.name, platform.split( '-' )[0] ),
|
||||
"%s.py" % self.name,
|
||||
"generic.py" )
|
||||
for build_script in build_scripts:
|
||||
build_script = os.path.join( ScrambleEgg.script_dir, build_script )
|
||||
if os.path.exists( build_script ):
|
||||
log.warning( "%s(): Using build script %s" % ( sys._getframe().f_code.co_name, build_script ) )
|
||||
break
|
||||
shutil.copyfile( build_script, os.path.join( self.buildpath, "scramble.py" ) )
|
||||
verfile = open( os.path.join( self.buildpath, ".galaxy_ver" ), "w" )
|
||||
verfile.write( self.version + '\n' )
|
||||
verfile.close()
|
||||
if self.tag is not None:
|
||||
tagfile = open( os.path.join( self.buildpath, ".galaxy_tag" ), "w" )
|
||||
tagfile.write( self.tag + '\n' )
|
||||
tagfile.close()
|
||||
if self.dependencies:
|
||||
depfile = open( os.path.join( self.buildpath, ".galaxy_deps" ), "w" )
|
||||
for dependency in self.dependencies:
|
||||
depfile.write( dependency + '\n' )
|
||||
depfile.close()
|
||||
|
||||
def run_scramble_script( self ):
|
||||
log.warning( "%s(): Beginning build" % sys._getframe().f_code.co_name )
|
||||
# subprocessed to sterilize the env
|
||||
cmd = "%s %s" % ( self.python, "scramble.py" )
|
||||
log.debug( '%s(): Executing in %s:' % ( sys._getframe().f_code.co_name, self.buildpath ) )
|
||||
log.debug( ' %s' % cmd )
|
||||
p = subprocess.Popen( args=cmd, shell=True, cwd=self.buildpath )
|
||||
r = p.wait()
|
||||
if r != 0:
|
||||
if sys.platform == 'sunos5' and get_config_var('CC').endswith('pycc') and not os.environ.get( 'PYCC_CC', None ):
|
||||
log.error( "%s(): Your python interpreter was compiled with Sun's pycc" % sys._getframe().f_code.co_name )
|
||||
log.error( " pseudo-compiler. You may need to set PYCC_CC and PYCC_CXX in your" )
|
||||
log.error( " environment if your compiler is in a non-standard location." )
|
||||
raise ScrambleFailure( self, "%s(): Egg build failed for %s %s" % ( sys._getframe().f_code.co_name, self.name, self.version ) )
|
||||
|
||||
|
||||
class ScrambleCrate( Crate ):
|
||||
"""
|
||||
Reads the eggs.ini file for use with scrambling eggs.
|
||||
"""
|
||||
def parse( self ):
|
||||
Crate.parse( self )
|
||||
# get dependency sources
|
||||
for egg in self.eggs.values():
|
||||
try:
|
||||
egg.sources = self.config.get( "source", egg.name ).split()
|
||||
except:
|
||||
egg.sources = []
|
||||
try:
|
||||
egg.dependencies = self.config.get( "dependencies", egg.name ).split()
|
||||
except:
|
||||
egg.dependencies = []
|
||||
|
||||
def parse_egg_section( self, *args, **kwargs ):
|
||||
kwargs['egg_class'] = ScrambleEgg
|
||||
Crate.parse_egg_section( self, *args, **kwargs )
|
||||
|
||||
def scramble( self, all=False ):
|
||||
if all:
|
||||
eggs = self.all_eggs
|
||||
else:
|
||||
eggs = self.config_eggs
|
||||
eggs = filter( lambda x: x.name not in self.no_auto, eggs )
|
||||
failed = []
|
||||
for egg in eggs:
|
||||
try:
|
||||
egg.scramble()
|
||||
except Exception, e:
|
||||
failed.append( egg )
|
||||
last_exc = e
|
||||
if failed:
|
||||
if len( failed ) == 1:
|
||||
raise last_exc # only 1 failure out of the crate, be more informative
|
||||
else:
|
||||
raise ScrambleFailure( failed )
|
||||
@@ -2,9 +2,6 @@
|
||||
Custom exceptions for Galaxy
|
||||
"""
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "Paste" )
|
||||
|
||||
from ..exceptions import error_codes
|
||||
|
||||
|
||||
|
||||
@@ -798,6 +798,10 @@ class JobWrapper( object ):
|
||||
self.__galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root
|
||||
return self.__galaxy_lib_dir
|
||||
|
||||
@property
|
||||
def galaxy_virtual_env(self):
|
||||
return os.environ.get('VIRTUAL_ENV', None)
|
||||
|
||||
# legacy naming
|
||||
get_job_runner = get_job_runner_url
|
||||
|
||||
|
||||
@@ -8,8 +8,6 @@ import logging
|
||||
import socket
|
||||
from galaxy.util import send_mail
|
||||
from galaxy.util.json import dumps
|
||||
from galaxy import eggs
|
||||
eggs.require( "MarkupSafe" )
|
||||
from markupsafe import escape
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -12,7 +12,6 @@ import subprocess
|
||||
|
||||
from Queue import Queue, Empty
|
||||
|
||||
import galaxy.eggs
|
||||
import galaxy.jobs
|
||||
from galaxy.jobs.command_factory import build_command
|
||||
from galaxy import model
|
||||
@@ -36,6 +35,7 @@ JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s'
|
||||
JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation"
|
||||
|
||||
GALAXY_LIB_ADJUST_TEMPLATE = """GALAXY_LIB="%s"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi;"""
|
||||
GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;"""
|
||||
|
||||
|
||||
class RunnerParams( ParamsWithSpecs ):
|
||||
@@ -254,12 +254,13 @@ class BaseJobRunner( object ):
|
||||
# so that long running set_meta()s can be canceled without having to reboot the server
|
||||
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and job_wrapper.output_paths:
|
||||
lib_adjust = GALAXY_LIB_ADJUST_TEMPLATE % job_wrapper.galaxy_lib_dir
|
||||
venv = GALAXY_VENV_TEMPLATE % job_wrapper.galaxy_virtual_env
|
||||
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames=job_wrapper.get_output_fnames(),
|
||||
set_extension=True,
|
||||
tmp_dir=job_wrapper.working_directory,
|
||||
# We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
|
||||
kwds={ 'overwrite' : False } )
|
||||
external_metadata_script = "%s %s" % (lib_adjust, external_metadata_script)
|
||||
external_metadata_script = "%s %s %s" % (lib_adjust, venv, external_metadata_script)
|
||||
if resolve_requirements:
|
||||
dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands()
|
||||
if dependency_shell_commands:
|
||||
@@ -275,20 +276,12 @@ class BaseJobRunner( object ):
|
||||
external_metadata_proc.wait()
|
||||
log.debug( 'execution of external set_meta for job %d finished' % job_wrapper.job_id )
|
||||
|
||||
def _get_egg_env_opts(self):
|
||||
env = []
|
||||
crate = galaxy.eggs.Crate()
|
||||
for opt in ('enable_egg_fetch', 'enable_eggs', 'try_dependencies_from_env'):
|
||||
env.append('GALAXY_%s="%s"; export GALAXY_%s' % (opt.upper(), getattr(crate, opt), opt.upper()))
|
||||
return env
|
||||
|
||||
def get_job_file(self, job_wrapper, **kwds):
|
||||
job_metrics = job_wrapper.app.job_metrics
|
||||
job_instrumenter = job_metrics.job_instrumenters[ job_wrapper.job_destination.id ]
|
||||
|
||||
env_setup_commands = kwds.get( 'env_setup_commands', [] )
|
||||
env_setup_commands.append( job_wrapper.get_env_setup_clause() or '' )
|
||||
env_setup_commands.extend( self._get_egg_env_opts() or [] )
|
||||
destination = job_wrapper.job_destination or {}
|
||||
envs = destination.get( "env", [] )
|
||||
envs.extend( job_wrapper.environment_variables )
|
||||
@@ -298,6 +291,7 @@ class BaseJobRunner( object ):
|
||||
options = dict(
|
||||
job_instrumenter=job_instrumenter,
|
||||
galaxy_lib=job_wrapper.galaxy_lib_dir,
|
||||
galaxy_virtual_env=job_wrapper.galaxy_virtual_env,
|
||||
env_setup_commands=env_setup_commands,
|
||||
working_directory=os.path.abspath( job_wrapper.working_directory ),
|
||||
command=command_line,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Job control via the DRMAA API.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
import json
|
||||
import logging
|
||||
@@ -9,21 +10,26 @@ import string
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from galaxy import eggs
|
||||
try:
|
||||
import drmaa
|
||||
DRMAA_IMPORT_MESSAGE = None
|
||||
except (ImportError, RuntimeError) as exc:
|
||||
drmaa = None
|
||||
DRMAA_IMPORT_MESSAGE = ('The Python drmaa package is required to use this '
|
||||
'feature, please install it or correct the '
|
||||
'following error:\n%s: %s' %
|
||||
(exc.__class__.__name__, str(exc)))
|
||||
|
||||
from galaxy import model
|
||||
from galaxy.jobs import JobDestination
|
||||
from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE
|
||||
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
|
||||
from galaxy.util import asbool
|
||||
|
||||
eggs.require( "drmaa" )
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
__all__ = [ 'DRMAAJobRunner' ]
|
||||
|
||||
drmaa = None
|
||||
|
||||
DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification',
|
||||
'workingDirectory', 'jobName', 'email', 'project' ]
|
||||
|
||||
@@ -36,8 +42,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
|
||||
|
||||
def __init__( self, app, nworkers, **kwargs ):
|
||||
"""Start the job runner"""
|
||||
|
||||
global drmaa
|
||||
assert drmaa is not None, DRMAA_IMPORT_MESSAGE
|
||||
|
||||
runner_param_specs = dict(
|
||||
drmaa_library_path=dict( map=str, default=os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ),
|
||||
@@ -57,10 +62,6 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
|
||||
log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path )
|
||||
os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path
|
||||
|
||||
# We foolishly named this file the same as the name exported by the drmaa
|
||||
# library... 'import drmaa' imports itself.
|
||||
drmaa = __import__( "drmaa" )
|
||||
|
||||
# Subclasses may need access to state constants
|
||||
self.drmaa_job_states = drmaa.JobState
|
||||
|
||||
|
||||
@@ -1,9 +1,20 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import logging
|
||||
import time
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
|
||||
try:
|
||||
import pbs
|
||||
PBS_IMPORT_MESSAGE = None
|
||||
except ImportError as exc:
|
||||
pbs = None
|
||||
PBS_IMPORT_MESSAGE = ('The Python pbs-python package is required to use '
|
||||
'this feature, please install it or correct the '
|
||||
'following error:\nImportError %s' % str(exc))
|
||||
|
||||
from galaxy import model
|
||||
from galaxy import util
|
||||
from galaxy.util.bunch import Bunch
|
||||
@@ -11,25 +22,6 @@ from galaxy.util import DATABASE_MAX_STRING_SIZE, shrink_stream_by_size
|
||||
from galaxy.jobs import JobDestination
|
||||
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
|
||||
|
||||
import pkg_resources
|
||||
|
||||
egg_message = """
|
||||
|
||||
The 'pbs' runner depends on 'pbs_python' which is not installed or not
|
||||
configured properly. Galaxy's "scramble" system should make this installation
|
||||
simple, please follow the instructions found at:
|
||||
|
||||
http://wiki.galaxyproject.org/Admin/Config/Performance/Cluster
|
||||
|
||||
Additional errors may follow:
|
||||
%s
|
||||
"""
|
||||
|
||||
try:
|
||||
pkg_resources.require( "pbs_python" )
|
||||
pbs = __import__( "pbs" )
|
||||
except Exception as e:
|
||||
raise Exception( egg_message % str( e ) )
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -99,8 +91,7 @@ class PBSJobRunner( AsynchronousJobRunner ):
|
||||
def __init__( self, app, nworkers ):
|
||||
"""Start the job runner """
|
||||
# Check if PBS was importable, fail if not
|
||||
if pbs is None:
|
||||
raise Exception( "PBSJobRunner requires pbs_python which was not found" )
|
||||
assert pbs is not None, PBS_IMPORT_MESSAGE
|
||||
if app.config.pbs_application_server and app.config.outputs_to_working_directory:
|
||||
raise Exception( "pbs_application_server (file staging) and outputs_to_working_directory options are mutually exclusive" )
|
||||
|
||||
|
||||
@@ -20,6 +20,11 @@ if [ "$GALAXY_LIB" != "None" ]; then
|
||||
export PYTHONPATH
|
||||
fi
|
||||
$env_setup_commands
|
||||
GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
|
||||
if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
|
||||
-a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then
|
||||
. "$GALAXY_VIRTUAL_ENV/bin/activate"
|
||||
fi
|
||||
$instrument_pre_commands
|
||||
cd $working_directory
|
||||
$command
|
||||
|
||||
@@ -15,6 +15,7 @@ GALAXY_SLOTS="1"
|
||||
REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path']
|
||||
OPTIONAL_TEMPLATE_PARAMS = {
|
||||
'galaxy_lib': None,
|
||||
'galaxy_virtual_env': None,
|
||||
'headers': '',
|
||||
'env_setup_commands': [],
|
||||
'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT,
|
||||
|
||||
@@ -25,12 +25,7 @@ attribute change to a model object.
|
||||
# instead of the three separate classes. With no 'apparent' perfect scheme
|
||||
# I'm opting to just keep them separate.
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( "SQLAlchemy >= 0.4" )
|
||||
import sqlalchemy
|
||||
|
||||
pkg_resources.require("repoze.lru") # used by Routes
|
||||
pkg_resources.require("Routes")
|
||||
import routes
|
||||
|
||||
from galaxy import exceptions
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
import pkg_resources
|
||||
|
||||
import functools
|
||||
import os
|
||||
import urllib2
|
||||
|
||||
pkg_resources.require('Beaker')
|
||||
from beaker.cache import CacheManager
|
||||
from beaker.util import parse_cache_config_options
|
||||
|
||||
|
||||
@@ -6,8 +6,6 @@ created (or copied) by users over the course of an analysis.
|
||||
"""
|
||||
import operator
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( "SQLAlchemy >= 0.4" )
|
||||
from sqlalchemy import desc, asc
|
||||
|
||||
from galaxy import model
|
||||
|
||||
@@ -3,8 +3,6 @@ Manager and Serializer for libraries.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import and_, false, not_, or_, true
|
||||
from sqlalchemy.orm.exc import MultipleResultsFound
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
Manager and Serializer for Users.
|
||||
"""
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( "SQLAlchemy >= 0.4" )
|
||||
import sqlalchemy
|
||||
|
||||
from galaxy import model
|
||||
|
||||
@@ -18,8 +18,6 @@ from itertools import ifilter
|
||||
from string import Template
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import and_, func, not_, or_, true, join, select
|
||||
from sqlalchemy.orm import joinedload, object_session, aliased
|
||||
from sqlalchemy.ext import hybrid
|
||||
|
||||
@@ -8,8 +8,6 @@ from sys import getsizeof
|
||||
from itertools import chain
|
||||
from collections import deque
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require("SQLAlchemy")
|
||||
import sqlalchemy
|
||||
|
||||
from galaxy import app
|
||||
|
||||
@@ -5,8 +5,6 @@ are encapsulated here.
|
||||
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import ( and_, asc, Boolean, Column, DateTime, desc, false, ForeignKey, Integer,
|
||||
MetaData, not_, Numeric, select, String, Table, TEXT, true, Unicode, UniqueConstraint )
|
||||
from sqlalchemy.ext.associationproxy import association_proxy
|
||||
|
||||
@@ -2,18 +2,9 @@ import sys
|
||||
import os.path
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "SQLAlchemy" )
|
||||
eggs.require( "six" ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "decorator" ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "Tempita " ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "sqlalchemy-migrate" )
|
||||
|
||||
from migrate.versioning import repository, schema
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from galaxy.model.orm import dialect_to_egg
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy.exc import NoSuchTableError
|
||||
from sqlalchemy import Table
|
||||
@@ -36,18 +27,6 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N
|
||||
3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
|
||||
4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
|
||||
"""
|
||||
dialect = ( url.split( ':', 1 ) )[0]
|
||||
try:
|
||||
egg = dialect_to_egg[dialect]
|
||||
try:
|
||||
eggs.require( egg )
|
||||
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
|
||||
except:
|
||||
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
|
||||
log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
|
||||
except KeyError:
|
||||
# Let this go, it could possibly work with db's we don't support
|
||||
log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
|
||||
# Create engine and metadata
|
||||
engine = create_engine( url, **engine_options )
|
||||
|
||||
|
||||
@@ -1,34 +1,3 @@
|
||||
import logging
|
||||
import pkg_resources
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
dialect_to_egg = {
|
||||
"sqlite": "pysqlite>=2",
|
||||
"postgres": "psycopg2",
|
||||
"postgresql": "psycopg2",
|
||||
"mysql": "MySQL_python"
|
||||
}
|
||||
|
||||
|
||||
def load_egg_for_url( url ):
|
||||
# Load the appropriate db module
|
||||
dialect = __guess_dialect_for_url( url )
|
||||
try:
|
||||
egg = dialect_to_egg[dialect]
|
||||
try:
|
||||
pkg_resources.require( egg )
|
||||
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
|
||||
except:
|
||||
# If the module's in the path elsewhere (i.e. non-egg), it'll still load.
|
||||
log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
|
||||
except KeyError:
|
||||
# Let this go, it could possibly work with db's we don't support
|
||||
log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
|
||||
|
||||
|
||||
def __guess_dialect_for_url( url ):
|
||||
return (url.split(':', 1))[0]
|
||||
"""
|
||||
galaxy.model.orm - ORM-related functionality
|
||||
"""
|
||||
|
||||
@@ -2,12 +2,9 @@ import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from galaxy.model.orm import load_egg_for_url
|
||||
|
||||
|
||||
def build_engine(url, engine_options, database_query_profiling_proxy=False, trace_logger=None):
|
||||
load_egg_for_url( url )
|
||||
|
||||
# Should we use the logging proxy?
|
||||
if database_query_profiling_proxy:
|
||||
import galaxy.model.orm.logging_connection_proxy as logging_connection_proxy
|
||||
|
||||
@@ -4,8 +4,6 @@ import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy.interfaces import ConnectionProxy
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -4,19 +4,8 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...).
|
||||
import logging
|
||||
import os.path
|
||||
|
||||
from galaxy import eggs
|
||||
|
||||
eggs.require( "decorator" )
|
||||
eggs.require( "Tempita" )
|
||||
eggs.require( "SQLAlchemy" )
|
||||
eggs.require( "six" ) # required by sqlalchemy-migrate
|
||||
eggs.require( "sqlparse" ) # required by sqlalchemy-migrate
|
||||
eggs.require( "sqlalchemy_migrate" )
|
||||
|
||||
from galaxy.util.properties import load_app_properties
|
||||
from galaxy.model.orm import dialect_to_egg
|
||||
|
||||
import pkg_resources
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -48,21 +37,6 @@ DATABASE = {
|
||||
}
|
||||
|
||||
|
||||
def require_dialect_egg( db_url ):
|
||||
dialect = ( db_url.split( ':', 1 ) )[0]
|
||||
try:
|
||||
egg = dialect_to_egg[dialect]
|
||||
try:
|
||||
pkg_resources.require( egg )
|
||||
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
|
||||
except:
|
||||
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
|
||||
log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
|
||||
except KeyError:
|
||||
# Let this go, it could possibly work with db's we don't support
|
||||
log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
|
||||
|
||||
|
||||
def read_config_file_arg( argv, default, old_default ):
|
||||
if '-c' in argv:
|
||||
pos = argv.index( '-c' )
|
||||
@@ -132,5 +106,4 @@ def get_config( argv, cwd=None ):
|
||||
else:
|
||||
db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % default_sqlite_file
|
||||
|
||||
require_dialect_egg( db_url )
|
||||
return dict(db_url=db_url, repo=repo, config_file=config_file, database=database)
|
||||
|
||||
@@ -27,8 +27,6 @@ select * from history where name='Unnamed history'
|
||||
import logging
|
||||
import re
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require("Parsley")
|
||||
import parsley
|
||||
|
||||
from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation,
|
||||
|
||||
@@ -2,14 +2,6 @@ import sys
|
||||
import os.path
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "SQLAlchemy" )
|
||||
eggs.require( "six" ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "decorator" ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "Tempita " ) # Required by sqlalchemy-migrate
|
||||
eggs.require( "sqlalchemy-migrate" )
|
||||
|
||||
# from sqlalchemy import *
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import MetaData
|
||||
@@ -18,8 +10,6 @@ from sqlalchemy import Table
|
||||
from sqlalchemy.exc import NoSuchTableError
|
||||
from migrate.versioning import repository, schema
|
||||
|
||||
from galaxy.model.orm import dialect_to_egg
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
# path relative to galaxy
|
||||
@@ -30,18 +20,6 @@ migrate_repository = repository.Repository( migrate_repository_directory )
|
||||
def create_or_verify_database( url, engine_options={}, app=None ):
|
||||
"""
|
||||
"""
|
||||
dialect = ( url.split( ':', 1 ) )[0]
|
||||
try:
|
||||
egg = dialect_to_egg[dialect]
|
||||
try:
|
||||
eggs.require( egg )
|
||||
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
|
||||
except:
|
||||
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
|
||||
log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
|
||||
except KeyError:
|
||||
# Let this go, it could possibly work with db's we don't support
|
||||
log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
|
||||
# Create engine and metadata
|
||||
engine = create_engine( url, **engine_options )
|
||||
|
||||
|
||||
@@ -15,17 +15,14 @@ from posixpath import dirname as path_dirname
|
||||
from galaxy.exceptions import ObjectNotFound
|
||||
from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs
|
||||
|
||||
try:
|
||||
import galaxy.eggs
|
||||
galaxy.eggs.require( 'PyRods' )
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import irods
|
||||
except ImportError:
|
||||
irods = None
|
||||
|
||||
NO_PYRODS_ERROR_MESSAGE = "IRODS object store configured, but no PyRods dependency available. Please install and properly configure PyRods or modify object store configuration."
|
||||
|
||||
IRODS_IMPORT_MESSAGE = ('The Python irods package is required to use this '
|
||||
'feature, please install it')
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -35,9 +32,8 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ):
|
||||
Galaxy object store based on iRODS
|
||||
"""
|
||||
def __init__( self, config, file_path=None, extra_dirs=None ):
|
||||
if irods is None:
|
||||
raise Exception(NO_PYRODS_ERROR_MESSAGE)
|
||||
super( IRODSObjectStore, self ).__init__( config, file_path=file_path, extra_dirs=extra_dirs )
|
||||
assert irods is not None, IRODS_IMPORT_MESSAGE
|
||||
self.cache_path = config.object_store_cache_path
|
||||
self.default_resource = config.irods_default_resource or None
|
||||
|
||||
|
||||
@@ -14,12 +14,6 @@ import subprocess
|
||||
|
||||
from multiprocessing.pool import IMapIterator
|
||||
|
||||
try:
|
||||
from galaxy import eggs
|
||||
eggs.require('boto')
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import boto
|
||||
from boto.s3.connection import S3Connection
|
||||
|
||||
@@ -5,16 +5,9 @@ reloading the toolbox, etc., across multiple processes.
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import sys
|
||||
|
||||
import galaxy.queues
|
||||
from galaxy import eggs, util
|
||||
eggs.require('anyjson')
|
||||
if sys.version_info < (2, 7, 0):
|
||||
# Kombu requires importlib and ordereddict to function under Python 2.6.
|
||||
eggs.require('importlib')
|
||||
eggs.require('ordereddict')
|
||||
eggs.require('kombu')
|
||||
from galaxy import util
|
||||
|
||||
from kombu import Connection
|
||||
from kombu.mixins import ConsumerMixin
|
||||
|
||||
@@ -4,18 +4,8 @@ All message queues used by Galaxy
|
||||
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
from galaxy.config import process_is_uwsgi
|
||||
|
||||
eggs.require('PyYAML')
|
||||
eggs.require('anyjson')
|
||||
if sys.version_info < (2, 7, 0):
|
||||
# Kombu requires importlib and ordereddict to function in Python 2.6.
|
||||
eggs.require('importlib')
|
||||
eggs.require('ordereddict')
|
||||
eggs.require('kombu')
|
||||
from kombu import Exchange, Queue, Connection
|
||||
|
||||
ALL_CONTROL = "control.*"
|
||||
|
||||
@@ -7,8 +7,6 @@ import operator
|
||||
import socket
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import and_, false, or_, not_
|
||||
from sqlalchemy.orm import eagerload_all
|
||||
|
||||
|
||||
@@ -12,13 +12,6 @@ import threading
|
||||
import urllib
|
||||
from datetime import datetime
|
||||
|
||||
from galaxy import eggs
|
||||
|
||||
eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako
|
||||
eggs.require( "Mako" )
|
||||
eggs.require( "Paste" )
|
||||
eggs.require( "SQLAlchemy >= 0.4" )
|
||||
|
||||
from cgi import FieldStorage
|
||||
from xml.etree import ElementTree
|
||||
from mako.template import Template
|
||||
|
||||
@@ -5,8 +5,6 @@ import subprocess
|
||||
import tempfile
|
||||
from cgi import FieldStorage
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy.orm import eagerload_all
|
||||
|
||||
from galaxy import datatypes, util
|
||||
|
||||
@@ -5,8 +5,6 @@ import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy.orm import eagerload, eagerload_all
|
||||
|
||||
from galaxy import model
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
try:
|
||||
from galaxy import eggs
|
||||
eggs.require("PyYAML")
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import yaml
|
||||
|
||||
from .yaml import YamlToolSource
|
||||
|
||||
@@ -2,10 +2,8 @@
|
||||
Module for building and searching the index of tools
|
||||
installed within this Galaxy.
|
||||
"""
|
||||
from galaxy import eggs
|
||||
from galaxy.web.framework.helpers import to_unicode
|
||||
|
||||
eggs.require( "Whoosh" )
|
||||
from whoosh.filedb.filestore import RamStorage
|
||||
from whoosh.fields import Schema, STORED, TEXT
|
||||
from whoosh.scoring import BM25F
|
||||
|
||||
@@ -5,10 +5,7 @@ import string
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "SQLAlchemy >= 0.4" )
|
||||
from sqlalchemy import and_
|
||||
eggs.require( "MarkupSafe" )
|
||||
from markupsafe import escape
|
||||
|
||||
from galaxy.model.item_attrs import Dictifiable
|
||||
|
||||
@@ -8,8 +8,7 @@ import os
|
||||
import string
|
||||
import sys
|
||||
import tempfile
|
||||
import pkg_resources
|
||||
pkg_resources.require( "bx-python" )
|
||||
|
||||
import bx.align.maf
|
||||
import bx.intervals
|
||||
import bx.interval_index_file
|
||||
|
||||
@@ -33,15 +33,11 @@ from os.path import relpath
|
||||
from hashlib import md5
|
||||
from itertools import izip
|
||||
|
||||
from galaxy import eggs
|
||||
|
||||
eggs.require( 'docutils' )
|
||||
import docutils.core
|
||||
import docutils.writers.html4css1
|
||||
|
||||
from xml.etree import ElementTree, ElementInclude
|
||||
|
||||
eggs.require( "wchartype" )
|
||||
import wchartype
|
||||
|
||||
from .inflection import Inflector, English
|
||||
|
||||
@@ -4,23 +4,19 @@ Provides a `TraceLogger` implementation that logs to a fluentd collector
|
||||
|
||||
import threading
|
||||
|
||||
try:
|
||||
import galaxy.eggs
|
||||
galaxy.eggs.require( "fluent-logger" )
|
||||
galaxy.eggs.require( "msgpack_python" )
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
from fluent.sender import FluentSender
|
||||
except ImportError:
|
||||
FluentSender = None
|
||||
|
||||
|
||||
FLUENT_IMPORT_MESSAGE = ('The Python fluent package is required to use this '
|
||||
'feature, please install it')
|
||||
|
||||
|
||||
class FluentTraceLogger( object ):
|
||||
def __init__( self, name, host='localhost', port=24224 ):
|
||||
if FluentSender is None:
|
||||
raise Exception("Attempted to use FluentTraceLogger with not Fluent dependency available.")
|
||||
assert FluentSender is not None, FLUENT_IMPORT_MESSAGE
|
||||
self.lock = threading.Lock()
|
||||
self.thread_local = threading.local()
|
||||
self.name = name
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
from xml.etree import ElementTree
|
||||
|
||||
try:
|
||||
from galaxy import eggs
|
||||
eggs.require('PyYAML')
|
||||
except Exception:
|
||||
# If not in Galaxy, ignore this.
|
||||
pass
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
import pkg_resources
|
||||
pkg_resources.require( "Cheetah" )
|
||||
|
||||
from Cheetah.Template import Template
|
||||
|
||||
|
||||
|
||||
@@ -9,14 +9,12 @@ import random
|
||||
import re
|
||||
import sys
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('numpy') # noqa
|
||||
eggs.require('bx-python') # noqa
|
||||
import pysam
|
||||
|
||||
from bx.interval_index_file import Indexes
|
||||
from bx.bbi.bigbed_file import BigBedFile
|
||||
from bx.bbi.bigwig_file import BigWigFile
|
||||
eggs.require('pysam') # noqa
|
||||
from pysam import csamtools, ctabix
|
||||
from pysam import ctabix
|
||||
|
||||
from galaxy.datatypes.interval import Bed, Gff, Gtf
|
||||
from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFFeature, GFFInterval, GFFReaderWrapper, parse_gff_attributes
|
||||
@@ -849,11 +847,11 @@ class BamDataProvider( GenomeDataProvider, FilterableMixin ):
|
||||
"""
|
||||
|
||||
# Open current BAM file using index.
|
||||
bamfile = csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb',
|
||||
index_filename=self.converted_dataset.file_name )
|
||||
bamfile = pysam.AlignmentFile( filename=self.original_dataset.file_name, mode='rb',
|
||||
index_filename=self.converted_dataset.file_name )
|
||||
|
||||
# TODO: write headers as well?
|
||||
new_bamfile = csamtools.Samfile( template=bamfile, filename=filename, mode='wb' )
|
||||
new_bamfile = pysam.AlignmentFile( template=bamfile, filename=filename, mode='wb' )
|
||||
|
||||
for region in regions:
|
||||
# Write data from region.
|
||||
@@ -881,7 +879,7 @@ class BamDataProvider( GenomeDataProvider, FilterableMixin ):
|
||||
|
||||
def open_data_file( self ):
|
||||
# Attempt to open the BAM file with index
|
||||
return csamtools.Samfile( filename=self.original_dataset.file_name, mode='rb',
|
||||
return pysam.AlignmentFile( filename=self.original_dataset.file_name, mode='rb',
|
||||
index_filename=self.converted_dataset.file_name )
|
||||
|
||||
def get_iterator( self, data_file, chrom, start, end, **kwargs ):
|
||||
|
||||
@@ -6,9 +6,6 @@ from a query string and render a webpage based on those data.
|
||||
import os
|
||||
import copy
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( 'MarkupSafe' )
|
||||
pkg_resources.require( 'Mako' )
|
||||
import mako
|
||||
|
||||
from galaxy.managers import api_keys
|
||||
|
||||
@@ -5,9 +5,6 @@ import logging
|
||||
import operator
|
||||
import re
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require("SQLAlchemy >= 0.4")
|
||||
|
||||
from sqlalchemy import true
|
||||
|
||||
from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError
|
||||
|
||||
@@ -3,8 +3,6 @@ import os
|
||||
from datetime import datetime, timedelta
|
||||
from string import punctuation as PUNCTUATION
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import and_, false, func, or_
|
||||
|
||||
import galaxy.queue_worker
|
||||
|
||||
@@ -16,9 +16,6 @@ from galaxy import util
|
||||
from galaxy.util import odict
|
||||
from galaxy.util import bunch
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( 'MarkupSafe' )
|
||||
pkg_resources.require( 'Mako' )
|
||||
import mako.lookup
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -2,8 +2,5 @@
|
||||
Galaxy web application framework
|
||||
"""
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "amqp" )
|
||||
|
||||
import base
|
||||
url_for = base.routes.url_for
|
||||
|
||||
@@ -9,13 +9,6 @@ import socket
|
||||
import tarfile
|
||||
import types
|
||||
|
||||
import pkg_resources
|
||||
|
||||
pkg_resources.require("Paste")
|
||||
pkg_resources.require("repoze.lru") # used by Routes
|
||||
pkg_resources.require("Routes")
|
||||
pkg_resources.require("WebOb")
|
||||
|
||||
import routes
|
||||
import webob
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@ import inspect
|
||||
from traceback import format_exc
|
||||
from functools import wraps
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "Paste" )
|
||||
import paste.httpexceptions
|
||||
|
||||
from galaxy.web.framework import url_for
|
||||
|
||||
@@ -4,16 +4,11 @@ Galaxy web framework helpers
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from galaxy import eggs
|
||||
from galaxy.util import hash_util
|
||||
from galaxy.util.json import safe_dumps as dumps # noqa (used by mako templates)
|
||||
eggs.require( "MarkupSafe" ) # required by WebHelpers
|
||||
eggs.require( "WebHelpers" )
|
||||
from webhelpers import date
|
||||
from webhelpers.html.tags import stylesheet_link, javascript_link
|
||||
|
||||
eggs.require( "repoze.lru" ) # used by Routes
|
||||
eggs.require( "Routes" )
|
||||
from routes import url_for
|
||||
|
||||
server_starttime = int(time.time())
|
||||
|
||||
@@ -5,21 +5,11 @@ information at the bottom of each page.
|
||||
|
||||
import threading
|
||||
import cgi
|
||||
import cProfile
|
||||
import pstats
|
||||
|
||||
from paste import response
|
||||
|
||||
try:
|
||||
# Included in Python 2.5
|
||||
import cProfile
|
||||
except:
|
||||
try:
|
||||
# Included in lsprof package for Python 2.4
|
||||
import pkg_resources
|
||||
pkg_resources.require( "lsprof" )
|
||||
import cProfile
|
||||
except:
|
||||
cProfile = None
|
||||
|
||||
import pstats
|
||||
|
||||
template = """
|
||||
<script>
|
||||
|
||||
@@ -6,11 +6,15 @@ raven.middleware
|
||||
:license: BSD, see LICENSE for more details.
|
||||
"""
|
||||
|
||||
import galaxy.eggs
|
||||
galaxy.eggs.require( "raven" )
|
||||
try:
|
||||
from raven import Client
|
||||
from raven.utils.wsgi import get_current_url, get_headers, get_environ
|
||||
except:
|
||||
Client = None
|
||||
|
||||
from raven import Client
|
||||
from raven.utils.wsgi import get_current_url, get_headers, get_environ
|
||||
|
||||
RAVEN_IMPORT_MESSAGE = ('The Python raven package is required to use this '
|
||||
'feature, please install it')
|
||||
|
||||
|
||||
class Sentry(object):
|
||||
@@ -19,6 +23,7 @@ class Sentry(object):
|
||||
uncaught exceptions and send them to Sentry.
|
||||
"""
|
||||
def __init__(self, application, dsn):
|
||||
assert Client is not None, RAVEN_IMPORT_MESSAGE
|
||||
self.application = application
|
||||
self.client = Client( dsn )
|
||||
|
||||
|
||||
@@ -6,24 +6,34 @@ import os
|
||||
import pickle
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( 'python-openid' )
|
||||
try:
|
||||
from openid import oidutil
|
||||
from openid.store import filestore
|
||||
from openid.consumer import consumer
|
||||
from openid.extensions import sreg
|
||||
except ImportError:
|
||||
oidutil = None
|
||||
|
||||
from openid import oidutil
|
||||
from openid.store import filestore
|
||||
from openid.consumer import consumer
|
||||
from openid.extensions import sreg
|
||||
class FakeConsumer( object ):
|
||||
__getattr__ = lambda x, y: None
|
||||
consumer = FakeConsumer()
|
||||
|
||||
|
||||
OPENID_IMPORT_MESSAGE = ('The Python openid package is required to use this '
|
||||
'feature, please install it')
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
|
||||
def oidlog( message, level=0 ):
|
||||
log.debug( message )
|
||||
oidutil.log = oidlog
|
||||
if oidutil is not None:
|
||||
oidutil.log = oidlog
|
||||
|
||||
|
||||
class OpenIDManager( object ):
|
||||
def __init__( self, cache_path ):
|
||||
assert oidutil is not None, OPENID_IMPORT_MESSAGE
|
||||
self.session_path = os.path.join( cache_path, 'session' )
|
||||
self.store_path = os.path.join( cache_path, 'store' )
|
||||
for dir in self.session_path, self.store_path:
|
||||
|
||||
@@ -10,18 +10,11 @@ import string
|
||||
import time
|
||||
from Cookie import CookieError
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "Cheetah" )
|
||||
from Cheetah.Template import Template
|
||||
eggs.require( "Mako" )
|
||||
import mako.runtime
|
||||
import mako.lookup
|
||||
# pytz is used by Babel.
|
||||
eggs.require( "pytz" )
|
||||
eggs.require( "Babel" )
|
||||
from babel.support import Translations
|
||||
from babel import Locale
|
||||
eggs.require( "SQLAlchemy >= 0.4" )
|
||||
from sqlalchemy import and_, true
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
@@ -3,9 +3,6 @@ import os
|
||||
import os.path
|
||||
import logging
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( "pycrypto" )
|
||||
|
||||
import galaxy.exceptions
|
||||
|
||||
from Crypto.Cipher import Blowfish
|
||||
|
||||
@@ -4,10 +4,6 @@ API operations on a history.
|
||||
.. seealso:: :class:`galaxy.model.History`
|
||||
"""
|
||||
|
||||
import pkg_resources
|
||||
pkg_resources.require( "Paste" )
|
||||
|
||||
pkg_resources.require( "SQLAlchemy >= 0.4" )
|
||||
from sqlalchemy import true, false
|
||||
|
||||
from galaxy import exceptions
|
||||
|
||||
@@ -7,8 +7,6 @@ API operations on a jobs.
|
||||
import json
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import and_, false, or_
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ API operations on Quota objects.
|
||||
import logging
|
||||
|
||||
from paste.httpexceptions import HTTPBadRequest
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import false, true
|
||||
|
||||
from galaxy import web, util
|
||||
|
||||
@@ -3,8 +3,6 @@ API operations on a sample tracking system.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import and_, false
|
||||
|
||||
from galaxy import web
|
||||
|
||||
@@ -3,8 +3,6 @@ API operations on Role objects.
|
||||
"""
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import false
|
||||
|
||||
from galaxy import web
|
||||
|
||||
@@ -4,8 +4,6 @@ API operations on User objects.
|
||||
|
||||
import logging
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import false, true
|
||||
|
||||
from galaxy import exceptions
|
||||
|
||||
@@ -24,8 +24,6 @@ from galaxy import util
|
||||
from galaxy.util import asbool
|
||||
from galaxy.util.properties import load_app_properties
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('Paste')
|
||||
from paste import httpexceptions
|
||||
|
||||
import logging
|
||||
@@ -683,7 +681,6 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
|
||||
# Interactive exception debugging, scary dangerous if publicly
|
||||
# accessible, if not enabled we'll use the regular error printing
|
||||
# middleware.
|
||||
eggs.require( "WebError" )
|
||||
from weberror import evalexception
|
||||
app = evalexception.EvalException( app, conf,
|
||||
templating_formatters=build_template_error_formatters() )
|
||||
|
||||
@@ -2,8 +2,6 @@ import logging
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import false, or_
|
||||
|
||||
import tool_shed.repository_types.util as rt_util
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
from galaxy import eggs
|
||||
eggs.require( "MarkupSafe" )
|
||||
from markupsafe import escape
|
||||
eggs.require( "Paste" )
|
||||
import paste.httpexceptions
|
||||
|
||||
import galaxy.queue_worker
|
||||
|
||||
@@ -2,12 +2,8 @@ import logging
|
||||
import os
|
||||
import urllib
|
||||
|
||||
from galaxy import eggs
|
||||
eggs.require( "MarkupSafe" )
|
||||
from markupsafe import escape
|
||||
eggs.require( "Paste" )
|
||||
import paste.httpexceptions
|
||||
eggs.require('SQLAlchemy')
|
||||
from sqlalchemy import false, true
|
||||
|
||||
from galaxy import datatypes, model, util, web
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user