From 04127ff3d5de59fb2aee8c7189e5e362c0f88fed Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 16 Apr 2008 20:53:27 +0000 Subject: [PATCH] DRMAA build scripts, eggs.ini entry. And a minor bugfix to eggs.require. --- eggs.ini | 3 + lib/galaxy/eggs/__init__.py | 13 ++- .../scramble/scripts/DRMAA_python-macosx.py | 104 ++++++++++++++++++ scripts/scramble/scripts/DRMAA_python.py | 102 +++++++++++++++++ 4 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 scripts/scramble/scripts/DRMAA_python-macosx.py create mode 100644 scripts/scramble/scripts/DRMAA_python.py diff --git a/eggs.ini b/eggs.ini index f2d7ff6447b..57f0c8a66b0 100644 --- a/eggs.ini +++ b/eggs.ini @@ -12,6 +12,7 @@ repository = http://eggs.g2.bx.psu.edu [eggs:platform] bx_python = 0.5.0 Cheetah = 1.0 +DRMAA_python = 0.2 MySQL_python = 1.2.2 pbs_python = 2.9.4 psycopg2 = 2.0.6 @@ -54,11 +55,13 @@ python_lzo = _static flup = .dev_r2311 bx_python = _dev_r414 nose = .dev_r101 +DRMAA_python = _6.1u4 ; source location, necessary for scrambling [source] bx_python = http://dist.g2.bx.psu.edu/bx-python_dist-r414.tar.bz2 Cheetah = http://umn.dl.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-1.0.tar.gz +DRMAA_python = http://gridengine.sunsource.net/files/documents/7/36/DRMAA-python-0.2.tar.gz MySQL_python = http://superb-west.dl.sourceforge.net/sourceforge/mysql-python/MySQL-python-1.2.2.tar.gz http://mysql.mirrors.pair.com/Downloads/MySQL-5.0/mysql-5.0.51a.tar.gz pbs_python = http://ftp.sara.nl/pub/outgoing/pbs_python-2.9.4.tar.gz http://www.clusterresources.com/downloads/torque/torque-%s.tar.gz psycopg2 = http://initd.org/pub/software/psycopg/PSYCOPG-2-0/psycopg2-2.0.6.tar.gz http://ftp8.us.postgresql.org/postgresql/source/v8.2.6/postgresql-8.2.6.tar.bz2 diff --git a/lib/galaxy/eggs/__init__.py b/lib/galaxy/eggs/__init__.py index 08809fa46a1..fe137a6c09f 100644 --- a/lib/galaxy/eggs/__init__.py +++ b/lib/galaxy/eggs/__init__.py @@ -425,10 +425,17 @@ def require( pkg ): # there's a conflicting egg on the pythonpath, remove it dist = pkg_resources.get_distribution( name ) working_set = pkg_resources.working_set - working_set.entries.remove( dist.location ) + # use the canonical path for comparisons + location = os.path.realpath( dist.location ) + for entry in working_set.entries: + if os.path.realpath( entry ) == location: + working_set.entries.remove( entry ) + break + else: + raise # some path weirdness has prevented us from finding the offender del working_set.by_key[dist.key] - working_set.entry_keys[dist.location] = [] - sys.path.remove(dist.location) + working_set.entry_keys[entry] = [] + sys.path.remove(entry) # get egg.find() if not egg.have: diff --git a/scripts/scramble/scripts/DRMAA_python-macosx.py b/scripts/scramble/scripts/DRMAA_python-macosx.py new file mode 100644 index 00000000000..993717c8186 --- /dev/null +++ b/scripts/scramble/scripts/DRMAA_python-macosx.py @@ -0,0 +1,104 @@ +import os, sys, subprocess, tarfile, shutil, urllib2 + +def unpack_sge(): + os.makedirs( "gridengine" ) + for file in [ SGE_COMMON_ARCHIVE, SGE_BINARY_PPC_ARCHIVE, SGE_BINARY_X86_ARCHIVE ]: + if not os.access( file, os.F_OK ): + fetch( file ) + t = tarfile.open( file, "r" ) + print "unpack_sge() Unpacking:" + print " ", file + for fn in t.getnames(): + t.extract( fn, "gridengine" ) + t.close() + +def lipo(): + darwin_dir = os.path.join( "gridengine", "lib", "darwin" ) + os.rename( "%s-ppc" % darwin_dir, "%s-ppc.orig" % darwin_dir ) + os.rename( "%s-x86" % darwin_dir, "%s-x86.orig" % darwin_dir ) + os.makedirs( darwin_dir ) + os.symlink( "darwin", "%s-ppc" % darwin_dir ) + os.symlink( "darwin", "%s-x86" % darwin_dir ) + ppc_libdrmaa = os.path.join( "%s-ppc.orig" % darwin_dir, "libdrmaa.dylib.1.0" ) + x86_libdrmaa = os.path.join( "%s-x86.orig" % darwin_dir, "libdrmaa.dylib.1.0" ) + fat_libdrmaa = os.path.join( darwin_dir, "libdrmaa.dylib.1.0" ) + lipo_cmd = "lipo %s %s -create -output %s" % ( ppc_libdrmaa, x86_libdrmaa, fat_libdrmaa ) + os.symlink( "libdrmaa.dylib.1.0", os.path.join( darwin_dir, "libdrmaa.dylib" ) ) + p = subprocess.Popen( args = lipo_cmd, shell = True ) + r = p.wait() + if r != 0: + print "lipo(): lipo failed" + sys.exit( 1 ) + +def fetch( file ): + url = SGE_BASE_URL + os.path.basename( file ) + print "fetch(): Fetching:" + print " ", url + inf = urllib2.urlopen( url ) + otf = open( file, 'wb' ) + otf.write( inf.read() ) + inf.close() + otf.close() + +# change back to the build dir +if os.path.dirname( sys.argv[0] ) != "": + os.chdir( os.path.dirname( sys.argv[0] ) ) + +# find setuptools +scramble_lib = os.path.join( "..", "..", "..", "lib" ) +sys.path.append( scramble_lib ) +try: + from setuptools import * + import pkg_resources +except: + from ez_setup import use_setuptools + use_setuptools( download_delay=8, to_dir=scramble_lib ) + from setuptools import * + import pkg_resources + +# get the tag +if os.access( ".galaxy_tag", os.F_OK ): + tagfile = open( ".galaxy_tag", "r" ) + tag = tagfile.readline().strip() +else: + tag = None + +SGE_VERSION = ( tag.split( "_" ) )[1] +SGE_BASE_URL = "http://gridengine.sunsource.net/download/SGE61/" +SGE_COMMON_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "ge-%s-common.tar.gz" %SGE_VERSION ) ) +SGE_BINARY_PPC_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "ge-%s-bin-darwin-ppc.tar.gz" % SGE_VERSION ) ) +SGE_BINARY_X86_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "ge-%s-bin-darwin-x86.tar.gz" % SGE_VERSION ) ) + +# clean, in case you're running this by hand from a dirty module source dir +for dir in [ "build", "dist", "gridengine" ]: + if os.access( dir, os.F_OK ): + print "scramble_it.py: removing dir:", dir + shutil.rmtree( dir ) + +# build/unpack SGE +unpack_sge() +lipo() + +file = "setup.py" +print "scramble(): Patching", file +if not os.access( "%s.orig" %file, os.F_OK ): + shutil.copyfile( file, "%s.orig" %file ) +i = open( "%s.orig" %file, "r" ) +o = open( file, "w" ) +for line in i.readlines(): + if line == 'SGE6_ROOT="/scratch_test02/SGE6"\n': + line = 'SGE6_ROOT="gridengine"\n' + print >>o, line, +i.close() +o.close() + +# tag +me = sys.argv[0] +sys.argv = [ me ] +if tag is not None: + sys.argv.append( "egg_info" ) + sys.argv.append( "--tag-build=%s" %tag ) +sys.argv.append( "bdist_egg" ) + +# go +execfile( "setup.py", globals(), locals() ) diff --git a/scripts/scramble/scripts/DRMAA_python.py b/scripts/scramble/scripts/DRMAA_python.py new file mode 100644 index 00000000000..34f33d5326a --- /dev/null +++ b/scripts/scramble/scripts/DRMAA_python.py @@ -0,0 +1,102 @@ +import os, sys, subprocess, tarfile, shutil, urllib2 + +def unpack_sge(): + os.makedirs( "gridengine" ) + for file in [ SGE_COMMON_ARCHIVE, SGE_BINARY_ARCHIVE ]: + if not os.access( file, os.F_OK ): + fetch( file ) + t = tarfile.open( file, "r" ) + print "unpack_sge() Unpacking:" + print " ", file + for fn in t.getnames(): + t.extract( fn, "gridengine" ) + t.close() + # i doubt you're using a 64 bit python, but if you are, this will break things + if arch == "sol-x86": + os.symlink( "sol-x86", os.path.join( "gridengine", "lib", "sol-amd64" ) ) + if arch == "sol-sparc64": + os.rename( os.path.join( "gridengine", "lib", "sol-sparc64" ), os.path.join( "gridengine", "lib", "sol-sparc64.orig" ) ) + os.symlink( "sol-sparc", os.path.join( "gridengine", "lib", "sol-sparc64" ) ) + +def fetch( file ): + url = SGE_BASE_URL + os.path.basename( file ) + print "fetch(): Fetching:" + print " ", url + inf = urllib2.urlopen( url ) + otf = open( file, 'wb' ) + otf.write( inf.read() ) + inf.close() + otf.close() + +# change back to the build dir +if os.path.dirname( sys.argv[0] ) != "": + os.chdir( os.path.dirname( sys.argv[0] ) ) + +# find setuptools +scramble_lib = os.path.join( "..", "..", "..", "lib" ) +sys.path.append( scramble_lib ) +try: + from setuptools import * + import pkg_resources +except: + from ez_setup import use_setuptools + use_setuptools( download_delay=8, to_dir=scramble_lib ) + from setuptools import * + import pkg_resources + +# get the tag +if os.access( ".galaxy_tag", os.F_OK ): + tagfile = open( ".galaxy_tag", "r" ) + tag = tagfile.readline().strip() +else: + tag = None + +SGE_VERSION = ( tag.split( "_" ) )[1] + +platform = pkg_resources.get_platform() +if platform.startswith( 'solaris' ): + if platform.endswith( 'i86pc' ): + arch = 'sol-x86' + elif platform.endswith( 'sun4u' ): + arch = 'sol-sparc64' +elif platform == 'linux-i686': + arch = 'lx24-x86' +elif platform == 'linux-x86_64': + arch = 'lx24-amd64' + +SGE_BASE_URL = "http://gridengine.sunsource.net/download/SGE61/" +SGE_COMMON_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "ge-%s-common.tar.gz" %SGE_VERSION ) ) +SGE_BINARY_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "ge-%s-bin-%s.tar.gz" %( SGE_VERSION, arch ) ) ) + +# clean, in case you're running this by hand from a dirty module source dir +for dir in [ "build", "dist", "gridengine" ]: + if os.access( dir, os.F_OK ): + print "scramble_it.py: removing dir:", dir + shutil.rmtree( dir ) + +# build/unpack SGE +unpack_sge() + +file = "setup.py" +print "scramble(): Patching", file +if not os.access( "%s.orig" %file, os.F_OK ): + shutil.copyfile( file, "%s.orig" %file ) +i = open( "%s.orig" %file, "r" ) +o = open( file, "w" ) +for line in i.readlines(): + if line == 'SGE6_ROOT="/scratch_test02/SGE6"\n': + line = 'SGE6_ROOT="gridengine"\n' + print >>o, line, +i.close() +o.close() + +# tag +me = sys.argv[0] +sys.argv = [ me ] +if tag is not None: + sys.argv.append( "egg_info" ) + sys.argv.append( "--tag-build=%s" %tag ) +sys.argv.append( "bdist_egg" ) + +# go +execfile( "setup.py", globals(), locals() )