opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 15:37:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Partially lint scripts/ .

Browse Source
This commit is contained in:
Nicola Soranzo
2015-07-13 20:13:28 +01:00
parent 29c4f1ae85
commit 6569c2dcc3
40 changed files with 514 additions and 498 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/pep8_sources.txt
+9 -1
View File
@@ -9,7 +9,15 @@ lib/galaxy/webapps/tool_shed/model/migrate/versions/
lib/galaxy_utils/
lib/pkg_resources.py
lib/tool_shed/
scripts/
scripts/api/
scripts/data_libraries/
scripts/loc_files/
scripts/microbes/
scripts/others/
scripts/scramble/
scripts/tool_shed/
scripts/tools/
scripts/transfer.py
test/base/
test/casperjs/
test/install_and_test_tool_shed_repositories/
scripts/build_toolbox.py
+20 -16
View File
@@ -1,13 +1,14 @@
import os
import sys
from xml.etree import ElementTree as ET
def prettify(elem):
from xml.dom import minidom
rough_string = ET.tostring(elem, 'utf-8')
repaired = minidom.parseString(rough_string)
return repaired.toprettyxml(indent=' ')
# Build a list of all toolconf xml files in the tools directory
def getfilenamelist(startdir):
filenamelist = []
@@ -32,6 +33,7 @@ def getfilenamelist(startdir):
print "DBG> tool config does not have a <section>:", fullfn
return filenamelist
class ToolBox(object):
def __init__(self):
from collections import defaultdict
@@ -39,21 +41,21 @@ class ToolBox(object):
self.sectionorders = {}
def add(self, toolelement, toolboxpositionelement):
section = toolboxpositionelement.attrib.get('section','')
label = toolboxpositionelement.attrib.get('label','')
section = toolboxpositionelement.attrib.get('section', '')
label = toolboxpositionelement.attrib.get('label', '')
order = int(toolboxpositionelement.attrib.get('order', '0'))
sectionorder = int(toolboxpositionelement.attrib.get('sectionorder', '0'))
# If this is the first time we encounter the section, store its order
# number. If we have seen it before, ignore the given order and use
# the stored one instead
if not self.sectionorders.has_key(section):
if section not in self.sectionorders:
self.sectionorders[section] = sectionorder
else:
sectionorder = self.sectionorders[section]
# Sortorder: add intelligent mix to the front
self.tools[("%05d-%s"%(sectionorder,section), label, order, section)].append(toolelement)
self.tools[("%05d-%s" % (sectionorder, section), label, order, section)].append(toolelement)
def addElementsTo(self, rootelement):
toolkeys = self.tools.keys()
@@ -76,7 +78,7 @@ class ToolBox(object):
if section:
sectionnumber += 1
attrib = {'name': section,
'id': "section%d"% sectionnumber}
'id': "section%d" % sectionnumber}
sectionelement = ET.Element('section', attrib)
rootelement.append(sectionelement)
currentelement = sectionelement
@@ -90,33 +92,34 @@ class ToolBox(object):
if label:
labelnumber += 1
attrib = {'text': label,
'id': "label%d"% labelnumber}
'id': "label%d" % labelnumber}
labelelement = ET.Element('label', attrib)
currentelement.append(labelelement)
# Add the tools that are in this place
for toolelement in self.tools[toolkey]:
currentelement.append(toolelement)
# Analyze all the toolconf xml files given in the filenamelist
# Analyze all the toolconf xml files given in the filenamelist
# Build a list of all sections
def scanfiles(filenamelist):
# Build an empty tool box
toolbox = ToolBox()
# Read each of the files in the list
for fn in filenamelist:
for fn in filenamelist:
doc = ET.parse(fn)
root = doc.getroot()
if root.tag == 'tool':
toolelements = [root]
else:
toolelements = doc.findall('tool')
for toolelement in toolelements:
# Figure out where the tool XML file is, absolute path.
if toolelement.attrib.has_key('file'):
if 'file' in toolelement.attrib:
# It is mentioned, we need to make it absolute
fileattrib = os.path.join(os.getcwd(),
os.path.dirname(fn),
@@ -136,7 +139,7 @@ def scanfiles(filenamelist):
tagarray.append(tag.text)
attrib['tags'] = ",".join(tagarray)
else:
print "DBG> No tags in",fn
print "DBG> No tags in", fn
# Build the tool element
newtoolelement = ET.Element('tool', attrib)
@@ -148,6 +151,7 @@ def scanfiles(filenamelist):
toolbox.add(newtoolelement, toolboxpositionelement)
return toolbox
def assemble():
filenamelist = []
for directorytree in ['tools']:
@@ -159,8 +163,8 @@ def assemble():
toolboxelement = ET.Element('toolbox')
toolbox.addElementsTo(toolboxelement)
print prettify(toolboxelement)
if __name__ == "__main__":
assemble()
scripts/build_universe_config.py
+3 -3
View File
@@ -20,10 +20,10 @@ def merge():
parser = ConfigParser()
for conf_file in conf_files:
parser.read([join(conf_directory, conf_file)])
## TODO: Expand enviroment variables here, that would
## also make Galaxy much easier to configure.
# TODO: Expand enviroment variables here, that would
# also make Galaxy much easier to configure.
destination= "config/galaxy.ini"
destination = "config/galaxy.ini"
if len(argv) > 2:
destination = argv[2]
scripts/check_eggs.py
+3 -1
View File
@@ -6,7 +6,9 @@ any are out of date.
usage: check_eggs.py [options]
"""
import os, sys, logging
import logging
import os
import sys
from optparse import OptionParser
parser = OptionParser()
scripts/check_galaxy.py
+41 -17
View File
@@ -4,11 +4,19 @@ check_galaxy can be run by hand, although it is meant to run from cron
via the check_galaxy.sh script in Galaxy's cron/ directory.
"""
import socket, sys, os, time, tempfile, filecmp, htmllib, formatter, getopt
import filecmp
import formatter
import getopt
import htmllib
import os
import socket
import sys
import tempfile
import time
from user import home
# options
if os.environ.has_key( "DEBUG" ):
if "DEBUG" in os.environ:
debug = os.environ["DEBUG"]
else:
debug = False
@@ -38,6 +46,7 @@ tools = {
]
}
# handle arg(s)
def usage():
print "usage: check_galaxy.py <server>"
@@ -92,8 +101,7 @@ except:
lib_dir = os.path.join( scripts_dir, "..", "lib" )
sys.path.insert( 1, lib_dir )
from galaxy import eggs
import pkg_resources
pkg_resources.require( "twill" )
eggs.require( "twill" )
import twill
import twill.commands as tc
@@ -104,6 +112,7 @@ socket.setdefaulttimeout(300)
tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.1)")
tc.config('use_tidy', 0)
class Browser:
def __init__(self):
@@ -138,7 +147,7 @@ class Browser:
p.feed(tc.browser.get_html())
if len(p.dids) > 0:
print "Remaining datasets ids:", " ".join( p.dids )
raise Exception, "History still contains datasets after attempting to delete them"
raise Exception("History still contains datasets after attempting to delete them")
if new_history:
self.get("/history/delete_current")
tc.save_cookies(self.cookie_jar)
@@ -168,12 +177,12 @@ class Browser:
# checks for a maint file
def check_maint(self):
if self.maint is None:
#dprint( "Warning: unable to check maint file for %s" % self.server )
# dprint( "Warning: unable to check maint file for %s" % self.server )
return(False)
try:
self.get(self.maint)
return(True)
except twill.errors.TwillAssertionError, e:
except twill.errors.TwillAssertionError:
return(False)
def login(self, user, pw):
@@ -190,9 +199,9 @@ class Browser:
dprint("user does not exist, will try creating")
self.create_user(user, pw)
elif p.bad_pw:
raise Exception, "Password is incorrect"
raise Exception("Password is incorrect")
else:
raise Exception, "Unknown error logging in"
raise Exception("Unknown error logging in")
tc.save_cookies(self.cookie_jar)
def create_user(self, user, pw):
@@ -206,12 +215,12 @@ class Browser:
p = userParser()
p.feed(tc.browser.get_html())
if p.already_exists:
raise Exception, 'The user you were trying to create already exists'
raise Exception('The user you were trying to create already exists')
def upload(self, file):
self.get("/tool_runner/index?tool_id=upload1")
tc.fv("1","file_type", "bed")
tc.formfile("1","file_data", file)
tc.fv("1", "file_type", "bed")
tc.formfile("1", "file_data", file)
tc.submit("runtool_btn")
tc.code(200)
@@ -236,17 +245,17 @@ class Browser:
else:
break
if count == maxiter:
raise Exception, "Tool never finished"
raise Exception("Tool never finished")
def check_status(self):
self.get("/root/history")
p = historyParser()
p.feed(tc.browser.get_html())
if p.status != "ok":
raise Exception, "JOB %s NOT OK: %s" % (p.id, p.status)
raise Exception("JOB %s NOT OK: %s" % (p.id, p.status))
self.id = p.id
self.status = p.status
#return((p.id, p.status))
# return((p.id, p.status))
def diff(self):
self.get("/datasets/%s/display/display?to_ext=bed" % self.id)
@@ -261,7 +270,7 @@ class Browser:
else:
if not debug:
os.remove(tmp[1])
raise Exception, "Tool output differs from expected"
raise Exception("Tool output differs from expected")
if not debug:
os.remove(tmp[1])
@@ -279,6 +288,7 @@ class Browser:
p.feed(tc.browser.get_html())
return p.logged_in
class userParser(htmllib.HTMLParser):
def __init__(self):
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
@@ -287,14 +297,19 @@ class userParser(htmllib.HTMLParser):
self.no_user = False
self.bad_pw = False
self.already_exists = False
def start_span(self, attrs):
self.in_span = True
def start_div(self, attrs):
self.in_div = True
def end_span(self):
self.in_span = False
def end_div(self):
self.in_div = False
def handle_data(self, data):
if self.in_span or self.in_div:
if data == "No such user (please note that login is case sensitive)":
@@ -304,11 +319,13 @@ class userParser(htmllib.HTMLParser):
elif data == "User with that email already exists":
self.already_exists = True
class historyParser(htmllib.HTMLParser):
def __init__(self):
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
self.status = None
self.id = None
def start_div(self, attrs):
# find the top history item
for i in attrs:
@@ -321,25 +338,31 @@ class historyParser(htmllib.HTMLParser):
if self.status is not None:
self.reset()
class didParser(htmllib.HTMLParser):
def __init__(self):
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
self.dids = []
def start_div(self, attrs):
for i in attrs:
if i[0] == "id" and i[1].startswith("historyItemContainer-"):
self.dids.append( i[1].rsplit("historyItemContainer-", 1)[1] )
dprint("got a dataset id: %s" % self.dids[-1])
class loggedinParser(htmllib.HTMLParser):
def __init__(self):
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
self.in_p = False
self.logged_in = False
def start_p(self, attrs):
self.in_p = True
def end_p(self):
self.in_p = False
def handle_data(self, data):
if self.in_p:
if data == "You are currently not logged in.":
@@ -347,6 +370,7 @@ class loggedinParser(htmllib.HTMLParser):
elif data.startswith( "You are currently logged in as " ):
self.logged_in = True
def dprint(str):
if debug:
print str
@@ -384,7 +408,7 @@ if __name__ == "__main__":
elif k == 'tool_run_options':
b.tool_opts = v
else:
raise Exception, "Unknown key in tools dict: %s" % k
raise Exception("Unknown key in tools dict: %s" % k)
b.runtool()
b.wait()
scripts/check_python.py
+2 -1
View File
@@ -3,7 +3,7 @@ If the current installed python version is not 2.6 to 2.7, prints an error
message to stderr and returns 1
"""
import os, sys
import sys
msg = """ERROR: Your Python version is: %s
Galaxy is currently supported on Python 2.6 and 2.7. To run Galaxy,
@@ -11,6 +11,7 @@ please download and install a supported version from python.org. If a
supported version is installed but is not your default, getgalaxy.org
contains instructions on how to force Galaxy to use a different version.""" % sys.version[:3]
def check_python():
try:
assert sys.version_info[:2] >= ( 2, 6 ) and sys.version_info[:2] <= ( 2, 7 )
scripts/cleanup_datasets/admin_cleanup_datasets.py
+23 -31
View File
@@ -36,35 +36,34 @@ Email Template Variables:
Author: Lance Parsons (lparsons@princeton.edu)
"""
import ConfigParser
import os
import sys
import shutil
import logging
import shutil
import sys
import time
from collections import defaultdict
from datetime import datetime, timedelta
from optparse import OptionParser
from time import strftime
from cleanup_datasets import CleanupDatasetsApplication
from galaxy import eggs
eggs.require('SQLAlchemy')
import sqlalchemy as sa
from sqlalchemy import and_, false
eggs.require("Mako")
from mako.template import Template
import galaxy.config
import galaxy.model.mapping
import galaxy.util
log = logging.getLogger()
log.setLevel(10)
log.addHandler(logging.StreamHandler(sys.stdout))
from cleanup_datasets import CleanupDatasetsApplication
import pkg_resources
#pkg_resources.require("SQLAlchemy >= 0.4")
pkg_resources.require("Mako")
from mako.template import Template
import time
import ConfigParser
from datetime import datetime, timedelta
from time import strftime
from optparse import OptionParser
import galaxy.config
import galaxy.model.mapping
import sqlalchemy as sa
from galaxy.model.orm import and_
import galaxy.util
assert sys.version_info[:2] >= (2, 4)
@@ -180,15 +179,15 @@ def administrative_delete_datasets(app, cutoff_time, cutoff_days,
(app.model.HistoryDatasetAssociation.table.c.id,
app.model.HistoryDatasetAssociation.table.c.deleted),
whereclause=and_(
app.model.Dataset.table.c.deleted == False,
app.model.Dataset.table.c.deleted == false(),
app.model.HistoryDatasetAssociation.table.c.update_time
< cutoff_time,
app.model.HistoryDatasetAssociation.table.c.deleted == False),
app.model.HistoryDatasetAssociation.table.c.deleted == false()),
from_obj=[sa.outerjoin(
app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table)])
# Add all datasets associated with Histories to our list
# Add all datasets associated with Histories to our list
hda_ids = []
hda_ids.extend(
[row.id for row in hda_ids_query.execute()])
@@ -249,15 +248,8 @@ def administrative_delete_datasets(app, cutoff_time, cutoff_days,
print "----------"
print msgtext
if not info_only:
#msg = MIMEText(msgtext)
#msg['Subject'] = subject
#msg['From'] = 'noone@nowhere.com'
#msg['To'] = email
galaxy.util.send_mail(fromaddr, email, subject,
msgtext, config)
#s = smtplib.SMTP(smtp_server)
#s.sendmail(['lparsons@princeton.edu'], email, msg.as_string())
#s.quit()
stop = time.time()
print ""
scripts/cleanup_datasets/cleanup_datasets.py
+126 -107
View File
@@ -1,38 +1,42 @@
#!/usr/bin/env python
import os, sys, logging
import ConfigParser
import logging
import os
import shutil
import sys
import time
from datetime import datetime, timedelta
from optparse import OptionParser
from time import strftime
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
eggs.require('SQLAlchemy')
import sqlalchemy as sa
from sqlalchemy import and_, false, null, true
from sqlalchemy.orm import eagerload
import galaxy.config
import galaxy.model.mapping
from galaxy.objectstore import build_object_store_from_config
from galaxy.exceptions import ObjectNotFound
log = logging.getLogger()
log.setLevel( 10 )
log.addHandler( logging.StreamHandler( sys.stdout ) )
from galaxy import eggs
import pkg_resources
pkg_resources.require( "SQLAlchemy >= 0.4" )
import time, ConfigParser, shutil
from datetime import datetime, timedelta
from time import strftime
from optparse import OptionParser
import galaxy.config
import galaxy.model.mapping
import sqlalchemy as sa
from galaxy.model.orm import and_, eagerload
from galaxy.objectstore import build_object_store_from_config
from galaxy.exceptions import ObjectNotFound
assert sys.version_info[:2] >= ( 2, 4 )
def main():
"""
Managing library datasets is a bit complex, so here is a scenario that hopefully provides clarification. The complexities
of handling library datasets is mostly contained in the delete_datasets() method in this script.
Assume we have 1 library dataset with: LibraryDatasetDatasetAssociation -> LibraryDataset and Dataset
At this point, we have the following database column values:
@@ -46,14 +50,14 @@ def main():
LibraryDatasetDatasetAssociation deleted: False
LibraryDataset deleted: True*, purged: False
Dataset deleted: False, purged: False
2. After the number of days configured for the delete_datasets() method (option -6 below) have passed, execution
of the delete_datasets() method results in the following database column values (changes from previous step marked with *):
LibraryDatasetDatasetAssociation deleted: True*
LibraryDataset deleted: True, purged: True*
Dataset deleted: True*, purged: False
3. After the number of days configured for the purge_datasets() method (option -3 below) have passed, execution
of the purge_datasets() method results in the following database column values (changes from previous step marked with *):
@@ -64,7 +68,7 @@ def main():
This scenario is about as simple as it gets. Keep in mind that a Dataset object can have many HistoryDatasetAssociations
and many LibraryDatasetDatasetAssociations, and a LibraryDataset can have many LibraryDatasetDatasetAssociations.
Another way of stating it is: LibraryDatasetDatasetAssociation objects map LibraryDataset objects to Dataset objects,
and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
"""
usage = "usage: %prog [options] galaxy.ini"
parser = OptionParser(usage=usage)
@@ -84,68 +88,69 @@ def main():
parser.print_help()
sys.exit()
ini_file = args[0]
if not ( options.purge_folders ^ options.delete_userless_histories ^ \
options.purge_libraries ^ options.purge_histories ^ \
if not ( options.purge_folders ^ options.delete_userless_histories ^
options.purge_libraries ^ options.purge_histories ^
options.purge_datasets ^ options.delete_datasets ):
parser.print_help()
sys.exit(0)
if options.remove_from_disk and options.info_only:
parser.error( "remove_from_disk and info_only are mutually exclusive" )
config_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} )
config_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} )
config_parser.read( ini_file )
config_dict = {}
for key, value in config_parser.items( "app:main" ):
config_dict[key] = value
config = galaxy.config.Configuration( **config_dict )
app = CleanupDatasetsApplication( config )
cutoff_time = datetime.utcnow() - timedelta( days=options.days )
now = strftime( "%Y-%m-%d %H:%M:%S" )
print "##########################################"
print "\n# %s - Handling stuff older than %i days" % ( now, options.days )
if options.info_only:
print "# Displaying info only ( --info_only )\n"
elif options.remove_from_disk:
print "Datasets will be removed from disk.\n"
else:
print "Datasets will NOT be removed from disk.\n"
if options.delete_userless_histories:
delete_userless_histories( app, cutoff_time, info_only = options.info_only, force_retry = options.force_retry )
delete_userless_histories( app, cutoff_time, info_only=options.info_only, force_retry=options.force_retry )
elif options.purge_histories:
purge_histories( app, cutoff_time, options.remove_from_disk, info_only = options.info_only, force_retry = options.force_retry )
purge_histories( app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry )
elif options.purge_datasets:
purge_datasets( app, cutoff_time, options.remove_from_disk, info_only = options.info_only, force_retry = options.force_retry )
purge_datasets( app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry )
elif options.purge_libraries:
purge_libraries( app, cutoff_time, options.remove_from_disk, info_only = options.info_only, force_retry = options.force_retry )
purge_libraries( app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry )
elif options.purge_folders:
purge_folders( app, cutoff_time, options.remove_from_disk, info_only = options.info_only, force_retry = options.force_retry )
purge_folders( app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry )
elif options.delete_datasets:
delete_datasets( app, cutoff_time, options.remove_from_disk, info_only = options.info_only, force_retry = options.force_retry )
delete_datasets( app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry )
app.shutdown()
sys.exit(0)
def delete_userless_histories( app, cutoff_time, info_only = False, force_retry = False ):
def delete_userless_histories( app, cutoff_time, info_only=False, force_retry=False ):
# Deletes userless histories whose update_time value is older than the cutoff_time.
# The purge history script will handle marking DatasetInstances as deleted.
# The purge history script will handle marking DatasetInstances as deleted.
# Nothing is removed from disk yet.
history_count = 0
start = time.time()
if force_retry:
histories = app.sa_session.query( app.model.History ) \
.filter( and_( app.model.History.table.c.user_id==None,
.filter( and_( app.model.History.table.c.user_id == null(),
app.model.History.table.c.update_time < cutoff_time ) )
else:
histories = app.sa_session.query( app.model.History ) \
.filter( and_( app.model.History.table.c.user_id==None,
app.model.History.table.c.deleted==False,
.filter( and_( app.model.History.table.c.user_id == null(),
app.model.History.table.c.deleted == false(),
app.model.History.table.c.update_time < cutoff_time ) )
for history in histories:
if not info_only:
@@ -157,9 +162,10 @@ def delete_userless_histories( app, cutoff_time, info_only = False, force_retry
stop = time.time()
print "Deleted %d histories" % history_count
print "Elapsed time: ", stop - start
print "##########################################"
print "##########################################"
def purge_histories( app, cutoff_time, remove_from_disk, info_only = False, force_retry = False ):
def purge_histories( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ):
# Purges deleted histories whose update_time is older than the cutoff_time.
# The dataset associations of each history are also marked as deleted.
# The Purge Dataset method will purge each Dataset as necessary
@@ -169,25 +175,25 @@ def purge_histories( app, cutoff_time, remove_from_disk, info_only = False, forc
start = time.time()
if force_retry:
histories = app.sa_session.query( app.model.History ) \
.filter( and_( app.model.History.table.c.deleted==True,
.filter( and_( app.model.History.table.c.deleted == true(),
app.model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) )
else:
histories = app.sa_session.query( app.model.History ) \
.filter( and_( app.model.History.table.c.deleted==True,
app.model.History.table.c.purged==False,
.filter( and_( app.model.History.table.c.deleted == true(),
app.model.History.table.c.purged == false(),
app.model.History.table.c.update_time < cutoff_time ) ) \
.options( eagerload( 'datasets' ) )
for history in histories:
print ("### Processing history id %d (%s)" % (history.id, history.name)).encode('utf-8')
for dataset_assoc in history.datasets:
_purge_dataset_instance( dataset_assoc, app, remove_from_disk, info_only = info_only ) #mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
_purge_dataset_instance( dataset_assoc, app, remove_from_disk, info_only=info_only ) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
if not info_only:
# TODO: should the Delete DefaultHistoryPermissions be deleted here? This was incorrectly
# done in the _list_delete() method of the history controller, so copied it here. Not sure
# done in the _list_delete() method of the history controller, so copied it here. Not sure
# if we should ever delete info like this from the db though, so commented out for now...
#for dhp in history.default_permissions:
# dhp.delete()
# for dhp in history.default_permissions:
# dhp.delete()
print "Purging history id ", history.id
history.purged = True
app.sa_session.add( history )
@@ -198,9 +204,10 @@ def purge_histories( app, cutoff_time, remove_from_disk, info_only = False, forc
stop = time.time()
print 'Purged %d histories.' % history_count
print "Elapsed time: ", stop - start
print "##########################################"
print "##########################################"
def purge_libraries( app, cutoff_time, remove_from_disk, info_only = False, force_retry = False ):
def purge_libraries( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ):
# Purges deleted libraries whose update_time is older than the cutoff_time.
# The dataset associations of each library are also marked as deleted.
# The Purge Dataset method will purge each Dataset as necessary
@@ -210,15 +217,15 @@ def purge_libraries( app, cutoff_time, remove_from_disk, info_only = False, forc
start = time.time()
if force_retry:
libraries = app.sa_session.query( app.model.Library ) \
.filter( and_( app.model.Library.table.c.deleted==True,
.filter( and_( app.model.Library.table.c.deleted == true(),
app.model.Library.table.c.update_time < cutoff_time ) )
else:
libraries = app.sa_session.query( app.model.Library ) \
.filter( and_( app.model.Library.table.c.deleted==True,
app.model.Library.table.c.purged==False,
.filter( and_( app.model.Library.table.c.deleted == true(),
app.model.Library.table.c.purged == false(),
app.model.Library.table.c.update_time < cutoff_time ) )
for library in libraries:
_purge_folder( library.root_folder, app, remove_from_disk, info_only = info_only )
_purge_folder( library.root_folder, app, remove_from_disk, info_only=info_only )
if not info_only:
print "Purging library id ", library.id
library.purged = True
@@ -228,9 +235,10 @@ def purge_libraries( app, cutoff_time, remove_from_disk, info_only = False, forc
stop = time.time()
print '# Purged %d libraries .' % library_count
print "Elapsed time: ", stop - start
print "##########################################"
print "##########################################"
def purge_folders( app, cutoff_time, remove_from_disk, info_only = False, force_retry = False ):
def purge_folders( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ):
# Purges deleted folders whose update_time is older than the cutoff_time.
# The dataset associations of each folder are also marked as deleted.
# The Purge Dataset method will purge each Dataset as necessary
@@ -240,49 +248,50 @@ def purge_folders( app, cutoff_time, remove_from_disk, info_only = False, force_
start = time.time()
if force_retry:
folders = app.sa_session.query( app.model.LibraryFolder ) \
.filter( and_( app.model.LibraryFolder.table.c.deleted==True,
.filter( and_( app.model.LibraryFolder.table.c.deleted == true(),
app.model.LibraryFolder.table.c.update_time < cutoff_time ) )
else:
folders = app.sa_session.query( app.model.LibraryFolder ) \
.filter( and_( app.model.LibraryFolder.table.c.deleted==True,
app.model.LibraryFolder.table.c.purged==False,
.filter( and_( app.model.LibraryFolder.table.c.deleted == true(),
app.model.LibraryFolder.table.c.purged == false(),
app.model.LibraryFolder.table.c.update_time < cutoff_time ) )
for folder in folders:
_purge_folder( folder, app, remove_from_disk, info_only = info_only )
_purge_folder( folder, app, remove_from_disk, info_only=info_only )
folder_count += 1
stop = time.time()
print '# Purged %d folders.' % folder_count
print "Elapsed time: ", stop - start
print "##########################################"
print "##########################################"
def delete_datasets( app, cutoff_time, remove_from_disk, info_only = False, force_retry = False ):
def delete_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ):
# Marks datasets as deleted if associated items are all deleted.
start = time.time()
if force_retry:
history_dataset_ids_query = sa.select( ( app.model.Dataset.table.c.id,
app.model.Dataset.table.c.state ),
whereclause = app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
from_obj = [ sa.outerjoin( app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table ) ] )
whereclause=app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
from_obj=[ sa.outerjoin( app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table ) ] )
library_dataset_ids_query = sa.select( ( app.model.LibraryDataset.table.c.id,
app.model.LibraryDataset.table.c.deleted ),
whereclause = app.model.LibraryDataset.table.c.update_time < cutoff_time,
from_obj = [ app.model.LibraryDataset.table ] )
else:
whereclause=app.model.LibraryDataset.table.c.update_time < cutoff_time,
from_obj=[ app.model.LibraryDataset.table ] )
else:
# We really only need the id column here, but sqlalchemy barfs when trying to select only 1 column
history_dataset_ids_query = sa.select( ( app.model.Dataset.table.c.id,
app.model.Dataset.table.c.state ),
whereclause = and_( app.model.Dataset.table.c.deleted == False,
app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
app.model.HistoryDatasetAssociation.table.c.deleted == True ),
from_obj = [ sa.outerjoin( app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table ) ] )
whereclause=and_( app.model.Dataset.table.c.deleted == false(),
app.model.HistoryDatasetAssociation.table.c.update_time < cutoff_time,
app.model.HistoryDatasetAssociation.table.c.deleted == true() ),
from_obj=[ sa.outerjoin( app.model.Dataset.table,
app.model.HistoryDatasetAssociation.table ) ] )
library_dataset_ids_query = sa.select( ( app.model.LibraryDataset.table.c.id,
app.model.LibraryDataset.table.c.deleted ),
whereclause = and_( app.model.LibraryDataset.table.c.deleted == True,
app.model.LibraryDataset.table.c.purged == False,
app.model.LibraryDataset.table.c.update_time < cutoff_time ),
from_obj = [ app.model.LibraryDataset.table ] )
whereclause=and_( app.model.LibraryDataset.table.c.deleted == true(),
app.model.LibraryDataset.table.c.purged == false(),
app.model.LibraryDataset.table.c.update_time < cutoff_time ),
from_obj=[ app.model.LibraryDataset.table ] )
deleted_dataset_count = 0
deleted_instance_count = 0
skip = []
@@ -300,7 +309,7 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only = False, forc
print "######### Processing LibraryDataset id:", library_dataset_id
# Get the LibraryDataset and the current LibraryDatasetDatasetAssociation objects
ld = app.sa_session.query( app.model.LibraryDataset ).get( library_dataset_id )
ldda = ld.library_dataset_dataset_association
ldda = ld.library_dataset_dataset_association
# Append the associated Dataset object's id to our list of dataset_ids
dataset_ids.append( ldda.dataset_id )
# Mark all of the LibraryDataset's associated LibraryDatasetDatasetAssociation objects' as deleted
@@ -338,9 +347,10 @@ def delete_datasets( app, cutoff_time, remove_from_disk, info_only = False, forc
stop = time.time()
print "Examined %d datasets, marked %d datasets and %d dataset instances (HDA) as deleted" % ( len( skip ), deleted_dataset_count, deleted_instance_count )
print "Total elapsed time: ", stop - start
print "##########################################"
print "##########################################"
def purge_datasets( app, cutoff_time, remove_from_disk, info_only = False, force_retry = False ):
def purge_datasets( app, cutoff_time, remove_from_disk, info_only=False, force_retry=False ):
# Purges deleted datasets whose update_time is older than cutoff_time. Files may or may
# not be removed from disk.
dataset_count = 0
@@ -348,18 +358,18 @@ def purge_datasets( app, cutoff_time, remove_from_disk, info_only = False, force
start = time.time()
if force_retry:
datasets = app.sa_session.query( app.model.Dataset ) \
.filter( and_( app.model.Dataset.table.c.deleted==True,
app.model.Dataset.table.c.purgable==True,
.filter( and_( app.model.Dataset.table.c.deleted == true(),
app.model.Dataset.table.c.purgable == true(),
app.model.Dataset.table.c.update_time < cutoff_time ) )
else:
datasets = app.sa_session.query( app.model.Dataset ) \
.filter( and_( app.model.Dataset.table.c.deleted==True,
app.model.Dataset.table.c.purgable==True,
app.model.Dataset.table.c.purged==False,
.filter( and_( app.model.Dataset.table.c.deleted == true(),
app.model.Dataset.table.c.purgable == true(),
app.model.Dataset.table.c.purged == false(),
app.model.Dataset.table.c.update_time < cutoff_time ) )
for dataset in datasets:
file_size = dataset.file_size
_purge_dataset( app, dataset, remove_from_disk, info_only = info_only )
_purge_dataset( app, dataset, remove_from_disk, info_only=info_only )
dataset_count += 1
try:
disk_space += file_size
@@ -370,15 +380,16 @@ def purge_datasets( app, cutoff_time, remove_from_disk, info_only = False, force
if remove_from_disk:
print 'Freed disk space: ', disk_space
print "Elapsed time: ", stop - start
print "##########################################"
print "##########################################"
def _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_children=True, info_only=False, is_deletable=False ):
# A dataset_instance is either a HDA or an LDDA. Purging a dataset instance marks the instance as deleted,
# A dataset_instance is either a HDA or an LDDA. Purging a dataset instance marks the instance as deleted,
# and marks the associated dataset as deleted if it is not associated with another active DatsetInstance.
if not info_only:
print "Marking as deleted: %s id %d (for dataset id %d)" % \
( dataset_instance.__class__.__name__, dataset_instance.id, dataset_instance.dataset.id )
dataset_instance.mark_deleted( include_children = include_children )
dataset_instance.mark_deleted( include_children=include_children )
dataset_instance.clear_associated_files()
app.sa_session.add( dataset_instance )
app.sa_session.flush()
@@ -394,15 +405,17 @@ def _purge_dataset_instance( dataset_instance, app, remove_from_disk, include_ch
print "Not deleting dataset ", dataset_instance.dataset.id, " (will be possibly deleted without 'info_only' mode)"
else:
print "Not deleting dataset %d (shared between multiple histories/libraries, at least one not deleted)" % dataset_instance.dataset.id
#need to purge children here
# need to purge children here
if include_children:
for child in dataset_instance.children:
_purge_dataset_instance( child, app, remove_from_disk, include_children = include_children, info_only = info_only )
_purge_dataset_instance( child, app, remove_from_disk, include_children=include_children, info_only=info_only )
def _dataset_is_deletable( dataset ):
#a dataset is deletable when it no longer has any non-deleted associations
# a dataset is deletable when it no longer has any non-deleted associations
return not bool( dataset.active_history_associations or dataset.active_library_associations )
def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletable=False ):
# Marks a base dataset as deleted, hdas/lddas associated with dataset can no longer be undeleted.
# Metadata files attached to associated dataset Instances is removed now.
@@ -411,14 +424,14 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab
else:
# Mark all associated MetadataFiles as deleted and purged and remove them from disk
metadata_files = []
#lets create a list of metadata files, then perform actions on them
# lets create a list of metadata files, then perform actions on them
for hda in dataset.history_associations:
for metadata_file in app.sa_session.query( app.model.MetadataFile ) \
.filter( app.model.MetadataFile.table.c.hda_id==hda.id ):
.filter( app.model.MetadataFile.table.c.hda_id == hda.id ):
metadata_files.append( metadata_file )
for ldda in dataset.library_associations:
for metadata_file in app.sa_session.query( app.model.MetadataFile ) \
.filter( app.model.MetadataFile.table.c.lda_id==ldda.id ):
.filter( app.model.MetadataFile.table.c.lda_id == ldda.id ):
metadata_files.append( metadata_file )
for metadata_file in metadata_files:
op_description = "marked as deleted"
@@ -433,7 +446,7 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab
print "Removing disk file ", metadata_file.file_name
os.unlink( metadata_file.file_name )
except Exception, e:
print "Error, exception: %s caught attempting to purge metadata file %s\n" %( str( e ), metadata_file.file_name )
print "Error, exception: %s caught attempting to purge metadata file %s\n" % ( str( e ), metadata_file.file_name )
metadata_file.purged = True
app.sa_session.add( metadata_file )
app.sa_session.flush()
@@ -449,7 +462,8 @@ def _delete_dataset( dataset, app, remove_from_disk, info_only=False, is_deletab
else:
print "Dataset %i will be deleted (without 'info_only' mode)" % ( dataset.id )
def _purge_dataset( app, dataset, remove_from_disk, info_only = False ):
def _purge_dataset( app, dataset, remove_from_disk, info_only=False ):
if dataset.deleted:
try:
if dataset.purgable and _dataset_is_deletable( dataset ):
@@ -461,7 +475,7 @@ def _purge_dataset( app, dataset, remove_from_disk, info_only = False ):
os.unlink( dataset.file_name )
# Remove associated extra files from disk if they exist
if dataset.extra_files_path and os.path.exists( dataset.extra_files_path ):
shutil.rmtree( dataset.extra_files_path ) #we need to delete the directory and its contents; os.unlink would always fail on a directory
shutil.rmtree( dataset.extra_files_path ) # we need to delete the directory and its contents; os.unlink would always fail on a directory
usage_users = []
for hda in dataset.history_associations:
if not hda.purged:
@@ -492,15 +506,16 @@ def _purge_dataset( app, dataset, remove_from_disk, info_only = False ):
else:
print "Error: '%s' has not previously been deleted, so it cannot be purged\n" % dataset.file_name
def _purge_folder( folder, app, remove_from_disk, info_only = False ):
def _purge_folder( folder, app, remove_from_disk, info_only=False ):
"""Purges a folder and its contents, recursively"""
for ld in folder.datasets:
print "Deleting library dataset id ", ld.id
ld.deleted = True
for ldda in [ld.library_dataset_dataset_association] + ld.expired_datasets:
_purge_dataset_instance( ldda, app, remove_from_disk, info_only = info_only ) #mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
_purge_dataset_instance( ldda, app, remove_from_disk, info_only=info_only ) # mark a DatasetInstance as deleted, clear associated files, and mark the Dataset as deleted if it is deletable
for sub_folder in folder.folders:
_purge_folder( sub_folder, app, remove_from_disk, info_only = info_only )
_purge_folder( sub_folder, app, remove_from_disk, info_only=info_only )
if not info_only:
# TODO: should the folder permissions be deleted here?
print "Purging folder id ", folder.id
@@ -508,6 +523,7 @@ def _purge_folder( folder, app, remove_from_disk, info_only = False ):
app.sa_session.add( folder )
app.sa_session.flush()
class CleanupDatasetsApplication( object ):
"""Encapsulates the state of a Universe application"""
def __init__( self, config ):
@@ -516,6 +532,7 @@ class CleanupDatasetsApplication( object ):
self.object_store = build_object_store_from_config( config )
# Setup the database engine and ORM
self.model = galaxy.model.mapping.init( config.file_path, config.database_connection, engine_options={}, create_tables=False, object_store=self.object_store )
@property
def sa_session( self ):
"""
@@ -524,7 +541,9 @@ class CleanupDatasetsApplication( object ):
to allow migration toward a more SQLAlchemy 0.4 style of use.
"""
return self.model.context.current
def shutdown( self ):
self.object_store.shutdown()
if __name__ == "__main__": main()
if __name__ == "__main__":
main()
scripts/cleanup_datasets/pgcleanup.py
+17 -15
View File
@@ -5,12 +5,12 @@ pgcleanup.py - A script for cleaning up datasets in Galaxy efficiently, by
PostgreSQL 9.1 or greater is required.
"""
import os
import sys
import shutil
import logging
import inspect
import datetime
import inspect
import logging
import os
import shutil
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
@@ -19,24 +19,26 @@ sys.path.insert(0, os.path.join(galaxy_root, 'lib'))
from galaxy import eggs
eggs.require('psycopg2')
eggs.require('SQLAlchemy')
import psycopg2
eggs.require('SQLAlchemy')
from sqlalchemy.engine.url import make_url
import galaxy.config
from galaxy.exceptions import ObjectNotFound
from galaxy.objectstore import build_object_store_from_config
from galaxy.util.bunch import Bunch
log = logging.getLogger()
class MetadataFile(Bunch):
pass
class Dataset(Bunch):
pass
class Cleanup(object):
def __init__(self):
self.options = None
@@ -90,8 +92,8 @@ class Cleanup(object):
def __load_config(self):
log.info('Reading config from %s' % self.options.config)
config_parser = ConfigParser(dict(here = os.getcwd(),
database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE'))
config_parser = ConfigParser(dict(here=os.getcwd(),
database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE'))
config_parser.read(self.options.config)
config_dict = {}
@@ -241,7 +243,7 @@ class Cleanup(object):
filename = self.object_store.get_filename(metadata_file, extra_dir='_metadata_files', extra_dir_at_root=True, alt_name="metadata_%d.dat" % id)
self._log('Removing from disk: %s' % filename, action_name)
except (ObjectNotFound, AttributeError), e:
log.error('Unable to get MetadataFile %s filename: %s' % (id, e))
log.error('Unable to get MetadataFile %s filename: %s' % (id, e))
return
if not self.options.dry_run:
@@ -259,7 +261,7 @@ class Cleanup(object):
This could probably be done more efficiently.
"""
log.info('Recalculating disk usage for users whose HistoryDatasetAssociations were purged')
log.info('Recalculating disk usage for users whose HistoryDatasetAssociations were purged')
for user_id in self.disk_accounting_user_ids:
@@ -341,7 +343,7 @@ class Cleanup(object):
Mark deleted all "anonymous" Histories (not owned by a registered user) that are older than the specified number of days.
"""
log.info('Marking deleted all userless Histories older than %i days' % self.options.days)
event_id = self._create_event()
sql = """
@@ -405,7 +407,7 @@ class Cleanup(object):
metadata_file.id AS id,
metadata_file.object_store_id AS object_store_id),
deleted_icda_ids
AS ( UPDATE implicitly_converted_dataset_association
AS ( UPDATE implicitly_converted_dataset_association
SET deleted = true%s
FROM purged_hda_ids
WHERE purged_hda_ids.id = implicitly_converted_dataset_association.hda_parent_id
@@ -515,7 +517,7 @@ class Cleanup(object):
metadata_file.id AS id,
metadata_file.object_store_id AS object_store_id),
deleted_icda_ids
AS ( UPDATE implicitly_converted_dataset_association
AS ( UPDATE implicitly_converted_dataset_association
SET deleted = true%s
FROM purged_hda_ids
WHERE purged_hda_ids.id = implicitly_converted_dataset_association.hda_parent_id
@@ -741,7 +743,7 @@ class Cleanup(object):
try:
filename = self.object_store.get_filename(dataset)
except (ObjectNotFound, AttributeError), e:
log.error('Unable to get Dataset %s filename: %s' % (tup[0], e))
log.error('Unable to get Dataset %s filename: %s' % (tup[0], e))
continue
try:
scripts/cleanup_datasets/populate_uuid.py
+8 -13
View File
@@ -7,30 +7,25 @@ Going forward, these ids will be generated for all new datasets. This
script fixes datasets that were generated before the change.
"""
import sys, os, ConfigParser
import galaxy.app
from galaxy.util.bunch import Bunch
import galaxy.datatypes.tabular
from galaxy.model.orm.scripts import get_config
from galaxy import eggs
from galaxy.model import mapping
import sys
import uuid
eggs.require( "SQLAlchemy" )
from sqlalchemy import *
from galaxy.model import mapping
from galaxy.model.orm.scripts import get_config
assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog) :
print "usage: %s galaxy.ini" % prog
print """
Populates blank uuid fields in datasets with randomly generated values.
Going forward, these ids will be generated for all new datasets. This
script fixes datasets that were generated before the change.
"""
def main():
if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" :
usage(sys.argv[0])
@@ -38,14 +33,14 @@ def main():
ini_file = sys.argv.pop(1)
config = get_config(ini_file)
model = mapping.init( ini_file, config['db_url'], create_tables = False )
model = mapping.init( ini_file, config['db_url'], create_tables=False )
for row in model.context.query( model.Dataset ):
if row.uuid is None:
row.uuid = uuid.uuid4()
print "Setting dataset:", row.id, " UUID to ", row.uuid
model.context.flush()
for row in model.context.query( model.Workflow ):
if row.uuid is None:
row.uuid = uuid.uuid4()
scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py
+7 -3
View File
@@ -4,10 +4,12 @@ Removes a dataset file ( which was first renamed by appending _purged to the fil
Usage: python remove_renamed_datasets_from_disk.py renamed.log
"""
import sys, os
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog) :
print "usage: %s file" % prog
print """
@@ -19,6 +21,7 @@ A log of files deleted is created in a file with the same name as that input but
with .removed.log appended.
"""
def main():
if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" :
usage(sys.argv[0])
@@ -26,7 +29,7 @@ def main():
infile = sys.argv[1]
outfile = infile + ".removed.log"
out = open( outfile, 'w' )
print >> out, "# The following renamed datasets have been removed from disk"
i = 0
removed_files = 0
@@ -39,7 +42,8 @@ def main():
removed_files += 1
except Exception, exc:
print >> out, "# Error, exception " + str( exc ) + " caught attempting to remove " + line
print >> out, "# Removed " + str( removed_files ) + " files"
print >> out, "# Removed " + str( removed_files ) + " files"
if __name__ == "__main__":
main()
scripts/cleanup_datasets/rename_purged_datasets.py
+6 -3
View File
@@ -4,10 +4,12 @@ Renames a dataset file by appending _purged to the file name so that it can late
Usage: python rename_purged_datasets.py purge.log
"""
import sys, os
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog) :
print "usage: %s file" % prog
print """
@@ -20,6 +22,7 @@ disk with remove_renamed_datasets_from_disk.py, by supplying it with a list of
them.
"""
def main():
if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help" :
usage(sys.argv[0])
@@ -27,7 +30,7 @@ def main():
infile = sys.argv[1]
outfile = infile + ".renamed.log"
out = open( outfile, 'w' )
print >> out, "# The following renamed datasets can be removed from disk"
i = 0
renamed_files = 0
@@ -41,7 +44,7 @@ def main():
renamed_files += 1
except Exception, exc:
print >> out, "# Error, exception " + str( exc ) + " caught attempting to rename " + purged_filename
print >> out, "# Renamed " + str( renamed_files ) + " files"
print >> out, "# Renamed " + str( renamed_files ) + " files"
if __name__ == "__main__":
main()
scripts/cleanup_datasets/update_dataset_size.py
+13 -8
View File
@@ -4,11 +4,15 @@ Updates dataset.size column.
Remember to backup your database before running.
"""
import sys, os, ConfigParser
import ConfigParser
import os
import sys
import galaxy.app
assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog) :
print "usage: %s galaxy.ini" % prog
print """
@@ -16,26 +20,27 @@ Updates the dataset.size column. Users are advised to backup the database before
running.
"""
def main():
if len(sys.argv) != 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help" :
if len(sys.argv) != 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
usage(sys.argv[0])
sys.exit()
ini_file = sys.argv.pop(1)
conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} )
conf_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} )
conf_parser.read( ini_file )
configuration = {}
for key, value in conf_parser.items( "app:main" ):
for key, value in conf_parser.items( "app:main" ):
configuration[key] = value
app = galaxy.app.UniverseApplication( global_conf = ini_file, **configuration )
#Step through Datasets, determining size on disk for each.
app = galaxy.app.UniverseApplication( global_conf=ini_file, **configuration )
# Step through Datasets, determining size on disk for each.
print "Determining the size of each dataset..."
for row in app.model.Dataset.table.select().execute():
purged = app.model.Dataset.get( row.id ).purged
file_size = app.model.Dataset.get( row.id ).file_size
if file_size is None and not purged:
size_on_disk = app.model.Dataset.get( row.id ).get_size()
print "Updating Dataset.%d with file_size: %d" %( row.id, size_on_disk )
print "Updating Dataset.%d with file_size: %d" % ( row.id, size_on_disk )
app.model.Dataset.table.update( app.model.Dataset.table.c.id == row.id ).execute( file_size=size_on_disk )
app.shutdown()
sys.exit(0)
scripts/cleanup_datasets/update_metadata.py
+20 -45
View File
@@ -1,18 +1,24 @@
#!/usr/bin/env python
#Dan Blankenberg
# Dan Blankenberg
"""
Updates metadata in the database to match rev 1891.
Remember to backup your database before running.
"""
import sys, os, ConfigParser
import ConfigParser
import os
import sys
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
sys.path.insert(0, os.path.join(galaxy_root, 'lib'))
import galaxy.app
from galaxy.util.bunch import Bunch
import galaxy.datatypes.tabular
assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog) :
print "usage: %s galaxy.ini" % prog
print """
@@ -21,65 +27,34 @@ Updates the metadata in the database to match rev 1981.
Remember to backup your database before running.
"""
def main():
if len(sys.argv) != 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help" :
if len(sys.argv) != 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help" :
usage(sys.argv[0])
sys.exit()
ini_file = sys.argv.pop(1)
conf_parser = ConfigParser.ConfigParser({'here':os.getcwd()})
conf_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
conf_parser.read(ini_file)
configuration = {}
for key, value in conf_parser.items("app:main"): configuration[key] = value
app = galaxy.app.UniverseApplication( global_conf = ini_file, **configuration )
#Step through Database, turning metadata bunches into dictionaries.
#print "Changing metadata bunches to dictionaries."
#for row in app.model.Dataset.table.select().execute():
# if isinstance (row.metadata, Bunch):
# print row.id
# app.model.Dataset.table.update(app.model.Dataset.table.c.id == row.id).execute( _metadata = row.metadata.__dict__ )
#Make sure all metadata is jsonified
#print "Rewriting all metadata to database, setting metadata dbkey, to ensure JSONified storage."
#for row in app.model.Dataset.table.select().execute():
# print row.id
# data = app.model.Dataset.get(row.id)
# dbkey = data.old_dbkey
# if not dbkey or data.metadata.dbkey not in ["?", ["?"], None, []]:
# dbkey = data.metadata.dbkey
# if not dbkey: dbkey = "?"
# #change dbkey then flush, then change to real value and flush, ensures that metadata is rewritten to database
# data.dbkey="~"
# data.flush()
# data.dbkey=dbkey
# data.flush()
#Search out tabular datatypes (and subclasses) and initialize metadata
for key, value in conf_parser.items("app:main"):
configuration[key] = value
app = galaxy.app.UniverseApplication( global_conf=ini_file, **configuration )
# Search out tabular datatypes (and subclasses) and initialize metadata
print "Seeking out tabular based files and initializing metadata"
for row in app.model.Dataset.table.select().execute():
data = app.model.Dataset.get(row.id)
if issubclass(type(data.datatype), type(app.datatypes_registry.get_datatype_by_extension('tabular'))):
print row.id, data.extension
#Call meta_data for all tabular files
#special case interval type where we do not want to overwrite chr, start, end, etc assignments
# Call meta_data for all tabular files
# special case interval type where we do not want to overwrite chr, start, end, etc assignments
if issubclass(type(data.datatype), type(app.datatypes_registry.get_datatype_by_extension('interval'))):
galaxy.datatypes.tabular.Tabular().set_meta(data)
else:
data.set_meta()
app.model.context.add( data )
app.model.context.flush()
#Search out maf datatypes and make sure that available species is set.
#print "Seeking out maf files and setting available species."
#for row in app.model.Dataset.table.select(app.model.Dataset.table.c.extension == 'maf').execute():
# print row.id
# sys.stdout.flush()
# data = app.model.Dataset.get(row.id)
# if data.missing_meta:
# data.set_meta() #Call maf set metadata method, setting available species
# data.flush()
app.shutdown()
sys.exit(0)
scripts/db_shell.py
+10 -13
View File
@@ -10,11 +10,13 @@
# You can also use this script as a library, for instance see https://gist.github.com/1979583
# TODO: This script overlaps a lot with manage_db.py and create_db.py,
# these should maybe be refactored to remove duplication.
import sys
import datetime
import decimal
import os.path
import sys
db_shell_path = __file__
new_path = [ os.path.join( os.path.dirname( db_shell_path ), os.path.pardir, "lib" ) ]
new_path = [ os.path.join( os.path.dirname( db_shell_path ), os.path.pardir, "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
@@ -29,20 +31,17 @@ db_url = get_config( sys.argv )['db_url']
# Setup DB scripting environment
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from sqlalchemy import * # noqa
from sqlalchemy.orm import * # noqa
from sqlalchemy.exc import * # noqa
from galaxy.model.mapping import init
sa_session = init( '/tmp/', db_url ).context
from galaxy.model import *
from galaxy.model import * # noqa
# Helper function for debugging sqlalchemy queries...
# http://stackoverflow.com/questions/5631078/sqlalchemy-print-the-actual-query
import decimal
import datetime
def printquery(statement, bind=None):
"""
print a query, with values filled in
@@ -54,8 +53,7 @@ def printquery(statement, bind=None):
if isinstance(statement, sqlalchemy.orm.Query):
if bind is None:
bind = statement.session.get_bind(
statement._mapper_zero_or_none()
)
statement._mapper_zero_or_none() )
statement = statement.statement
elif bind is None:
bind = statement.bind
@@ -104,4 +102,3 @@ def printquery(statement, bind=None):
compiler = LiteralCompiler(dialect, statement)
print compiler.process(statement)
scripts/dist-scramble.py
+3 -1
View File
@@ -31,7 +31,9 @@ subdirectory of your Galaxy distribution. These eggs can then be copied to your
distribution site.
"""
import os, sys, logging
import logging
import os
import sys
from optparse import OptionParser
parser = OptionParser()
scripts/drmaa_external_killer.py
+12 -20
View File
@@ -5,34 +5,29 @@ Terminates a DRMAA job if given a job id and (appropriate) user id.
"""
import errno
import json
import os
import pwd
import sys
#import drmaa
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
import pkg_resources
pkg_resources.require("drmaa")
eggs.require("drmaa")
import drmaa
def validate_paramters():
if len(sys.argv)<3:
if len(sys.argv) < 3:
sys.stderr.write("usage: %s [job ID] [user uid]\n" % sys.argv[0])
exit(1)
jobID = sys.argv[1]
jobID = sys.argv[1]
uid = int(sys.argv[2])
return jobID, uid
def set_user(uid):
try:
gid = pwd.getpwuid(uid).pw_gid
@@ -44,24 +39,21 @@ def set_user(uid):
exit(1)
else:
pass
if os.getuid()==0:
if os.getuid() == 0:
sys.stderr.write("error: UID is 0 (root) after changing user. This script should not be run as root. aborting.\n" )
exit(1)
if os.geteuid()==0:
if os.geteuid() == 0:
sys.stderr.write("error: EUID is 0 (root) after changing user. This script should not be run as root. aborting.\n" )
exit(1)
def main():
jobID, uid = validate_paramters()
jobID, uid = validate_paramters()
set_user(uid)
s=drmaa.Session()
s = drmaa.Session()
s.initialize()
s.control(jobID,drmaa.JobControlAction.TERMINATE)
s.control(jobID, drmaa.JobControlAction.TERMINATE)
s.exit()
if __name__ == "__main__":
main()
scripts/drmaa_external_runner.py
+4 -6
View File
@@ -6,20 +6,18 @@ defining any or all of the following: args, remoteCommand, outputPath,
errorPath, nativeSpecification, name, email, project
"""
import os
import sys
import errno
import pwd
import json
import os
import pwd
import sys
#import drmaa
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
import pkg_resources
pkg_resources.require("drmaa")
eggs.require("drmaa")
import drmaa
DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification',
scripts/edam_mapping.py
+5 -3
View File
@@ -17,12 +17,14 @@ from __future__ import print_function
import os
import urllib2
import sys
from xml import etree
# Setup model, paths, etc...
import db_shell
new_path = [ os.path.join( os.path.dirname( __file__ ), os.path.pardir, "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
import galaxy.model
import galaxy.datatypes.registry
SCRIPTS_DIR = os.path.dirname(__file__)
scripts/egg_packager_template.py
+40 -18
View File
@@ -2,7 +2,16 @@
# Configure stdout logging
import os, sys, logging, glob, zipfile, shutil
import glob
import HTMLParser
import logging
import os
import re
import shutil
import sys
import urllib
import urllib2
import zipfile
log = logging.getLogger()
log.setLevel( 10 )
@@ -10,21 +19,20 @@ log.addHandler( logging.StreamHandler( sys.stdout ) )
# Fake pkg_resources
import re
macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)")
darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)")
solarisVersionString = re.compile(r"solaris-(\d)\.(\d+)-(.*)")
def compatible_platforms(provided,required):
def compatible_platforms(provided, required):
"""Can code for the `provided` platform run on the `required` platform?
Returns true if either platform is ``None``, or the platforms are equal.
XXX Needs compatibility checks for Linux and other unixy OSes.
"""
if provided is None or required is None or provided==required:
return True # easy case
if provided is None or required is None or provided == required:
return True # easy case
# Mac OS X special cases
reqMac = macosVersionString.match(required)
@@ -41,10 +49,10 @@ def compatible_platforms(provided,required):
dversion = int(provDarwin.group(1))
macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2))
if dversion == 7 and macosversion >= "10.3" or \
dversion == 8 and macosversion >= "10.4":
dversion == 8 and macosversion >= "10.4":
#import warnings
#warnings.warn("Mac eggs should be rebuilt to "
# import warnings
# warnings.warn("Mac eggs should be rebuilt to "
# "use the macosx designation instead of darwin.",
# category=DeprecationWarning)
return True
@@ -52,11 +60,9 @@ def compatible_platforms(provided,required):
# are they the same major version and machine type?
if provMac.group(1) != reqMac.group(1) or \
provMac.group(3) != reqMac.group(3):
provMac.group(3) != reqMac.group(3):
return False
# is the required OS major update >= the provided one?
if int(provMac.group(2)) > int(reqMac.group(2)):
return False
@@ -70,7 +76,7 @@ def compatible_platforms(provided,required):
if not provSol:
return False
if provSol.group(1) != reqSol.group(1) or \
provSol.group(3) != reqSol.group(3):
provSol.group(3) != reqSol.group(3):
return False
if int(provSol.group(2)) > int(reqSol.group(2)):
return False
@@ -85,6 +91,7 @@ EGG_NAME = re.compile(
re.VERBOSE | re.IGNORECASE
).match
class Distribution( object ):
def __init__( self, egg_name, project_name, version, py_version, platform ):
self._egg_name = egg_name
@@ -97,18 +104,21 @@ class Distribution( object ):
self.py_version = py_version
self.platform = platform
self.location = os.path.join( tmpd, egg_name ) + '.egg'
def egg_name( self ):
return self._egg_name
@classmethod
def from_filename( cls, basename ):
project_name, version, py_version, platform = [None]*4
project_name, version, py_version, platform = [None] * 4
basename, ext = os.path.splitext(basename)
if ext.lower() == '.egg':
match = EGG_NAME( basename )
if match:
project_name, version, py_version, platform = match.group( 'name','ver','pyver','plat' )
project_name, version, py_version, platform = match.group( 'name', 'ver', 'pyver', 'plat' )
return cls( basename, project_name, version, py_version, platform )
class pkg_resources( object ):
pass
@@ -117,25 +127,32 @@ pkg_resources.Distribution = Distribution
# Fake galaxy.eggs
env = None
def get_env():
return None
import urllib, urllib2, HTMLParser
class URLRetriever( urllib.FancyURLopener ):
def http_error_default( *args ):
urllib.URLopener.http_error_default( *args )
class Egg( object ):
def __init__( self, distribution ):
self.url = url + '/' + distribution.project_name.replace( '-', '_' )
self.dir = tmpd
self.distribution = distribution
def set_distribution( self ):
pass
def unpack_if_needed( self ):
pass
def remove_doppelgangers( self ):
pass
def fetch( self, requirement ):
"""
fetch() serves as the install method to pkg_resources.working_set.resolve()
@@ -151,6 +168,7 @@ class Egg( object ):
def __init__( self ):
HTMLParser.HTMLParser.__init__( self )
self.links = []
def handle_starttag( self, tag, attrs ):
if tag == 'a' and 'href' in dict( attrs ):
self.links.append( dict( attrs )['href'] )
@@ -197,9 +215,10 @@ class Egg( object ):
self.unpack_if_needed()
self.remove_doppelgangers()
global env
env = get_env() # reset the global Environment object now that we've obtained a new egg
env = get_env() # reset the global Environment object now that we've obtained a new egg
return self.distribution
def create_zip():
fname = 'galaxy_eggs-%s.zip' % platform
z = zipfile.ZipFile( fname, 'w', zipfile.ZIP_STORED )
@@ -211,6 +230,7 @@ def create_zip():
print "directory and unpack with:"
print " unzip %s" % fname
def clean():
shutil.rmtree( tmpd )
@@ -218,6 +238,8 @@ import tempfile
tmpd = tempfile.mkdtemp()
failures = []
platform = None
py = None
url = None
# Automatically generated egg definitions follow
scripts/external_chown_script.py
+9 -22
View File
@@ -1,37 +1,24 @@
#!/usr/bin/env python
import errno
import json
import os
import pwd
import sys
#import drmaa
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
import pkg_resources
pkg_resources.require("drmaa")
import drmaa
def validate_paramters():
if len(sys.argv)<4:
if len(sys.argv) < 4:
sys.stderr.write("usage: %s path user_name gid\n" % sys.argv[0])
exit(1)
path = sys.argv[1]
galaxy_user_name = sys.argv[2]
gid = sys.argv[3]
path = sys.argv[1]
galaxy_user_name = sys.argv[2]
gid = sys.argv[3]
return path, galaxy_user_name, gid
return path, galaxy_user_name, gid
def main():
path, galaxy_user_name, gid = validate_paramters()
os.system('chown -Rh %s %s' %(galaxy_user_name, path))
os.system('chgrp -Rh %s %s' %(gid, path))
path, galaxy_user_name, gid = validate_paramters()
os.system('chown -Rh %s %s' % (galaxy_user_name, path))
os.system('chgrp -Rh %s %s' % (gid, path))
if __name__ == "__main__":
main()
scripts/extract_dataset_part.py
+11 -10
View File
@@ -10,16 +10,17 @@ import json
import logging
import os
import sys
logging.basicConfig()
log = logging.getLogger( __name__ )
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
# This junk is here to prevent loading errors
import galaxy.model.mapping #need to load this before we unpickle, in order to setup properties assigned by the mappers
galaxy.model.Job() #this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here
import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers
galaxy.model.Job() # this looks REAL stupid, but it is REQUIRED in order for SA to insert parameters into the classes defined by the mappers --> it appears that instantiating ANY mapper'ed class would suffice here
logging.basicConfig()
log = logging.getLogger( __name__ )
def __main__():
"""
@@ -27,20 +28,20 @@ def __main__():
"""
file_path = sys.argv.pop( 1 )
if not os.path.isfile(file_path):
#Nothing to do - some splitters don't write a JSON file
# Nothing to do - some splitters don't write a JSON file
sys.exit(0)
data = json.load(open(file_path, 'r'))
try:
class_name_parts = data['class_name'].split('.')
module_name = '.'.join(class_name_parts[:-1])
class_name = class_name_parts[-1]
mod = __import__(module_name, globals(), locals(), [class_name])
cls = getattr(mod, class_name)
mod = __import__(module_name, globals(), locals(), [class_name])
cls = getattr(mod, class_name)
if not cls.process_split_file(data):
sys.stderr.write('Writing split file failed\n')
sys.exit(1)
except Exception, e:
sys.stderr.write(str(e))
sys.exit(1)
__main__()
scripts/extract_toolbox_sections.py
+31 -36
View File
@@ -1,22 +1,21 @@
import os
import sys
from xml.etree import ElementTree as ET
from collections import defaultdict
from xml.etree import ElementTree as ET
# Todo: ""
# execute from galaxy root dir
tooldict = defaultdict(list)
def main():
doc = ET.parse("tool_conf.xml")
root = doc.getroot()
# index range 1-1000, current sections/tools divided between 250-750
sectionindex = 250
sectionfactor = int( 500 / len( root.getchildren() ) )
for rootchild in root.getchildren():
currentsectionlabel = ""
if ( rootchild.tag == "section" ):
@@ -24,27 +23,26 @@ def main():
# per section tool index range 1-1000, current labels/tools
# divided between 20 and 750
toolindex = 250
toolfactor = int( 500 / len( rootchild.getchildren() ) )
toolfactor = int( 500 / len( rootchild.getchildren() ) )
currentlabel = ""
for sectionchild in rootchild.getchildren():
if ( sectionchild.tag == "tool" ):
addToToolDict(sectionchild, sectionname, sectionindex, toolindex, currentlabel)
toolindex += toolfactor
elif ( sectionchild.tag == "label" ):
currentlabel = sectionchild.attrib["text"]
sectionindex += sectionfactor
currentlabel = sectionchild.attrib["text"]
sectionindex += sectionfactor
elif ( rootchild.tag == "tool" ):
addToToolDict(rootchild, "", sectionindex, None, currentsectionlabel)
sectionindex += sectionfactor
elif ( rootchild.tag == "label" ):
currentsectionlabel = rootchild.attrib["text"]
sectionindex += sectionfactor
sectionindex += sectionfactor
elif ( rootchild.tag == "label" ):
currentsectionlabel = rootchild.attrib["text"]
sectionindex += sectionfactor
# scan galaxy root tools dir for tool-specific xmls
toolconffilelist = getfnl( os.path.join(os.getcwd(), "tools" ) )
# foreach tool xml:
# foreach tool xml:
# check if the tags element exists in the tool xml (as child of <tool>)
# if not, add empty tags element for later use
# if this tool is in the above tooldict, add the toolboxposition element to the tool xml
@@ -52,12 +50,12 @@ def main():
for toolconffile in toolconffilelist:
hastags = False
hastoolboxpos = False
#parse tool config file into a document structure as defined by the ElementTree
# parse tool config file into a document structure as defined by the ElementTree
tooldoc = ET.parse(toolconffile)
# get the root element of the toolconfig file
tooldocroot = tooldoc.getroot()
#check tags element, set flag
# check tags element, set flag
tagselement = tooldocroot.find("tags")
if (tagselement):
hastags = True
@@ -65,12 +63,12 @@ def main():
toolboxposelement = tooldocroot.find("toolboxposition")
if ( toolboxposelement ):
hastoolboxpos = True
if ( not ( hastags and hastoolboxpos ) ):
original = open( toolconffile, 'r' )
contents = original.readlines()
original.close()
# the new elements will be added directly below the root tool element
addelementsatposition = 1
# but what's on the first line? Root or not?
@@ -79,24 +77,22 @@ def main():
newelements = []
if ( not hastoolboxpos ):
if ( toolconffile in tooldict ):
for attributes in tooldict[toolconffile]:
# create toolboxposition element
sectionelement = ET.Element("toolboxposition")
sectionelement.attrib = attributes
sectionelement.tail = "\n "
newelements.append( ET.tostring(sectionelement, 'utf-8') )
for attributes in tooldict[toolconffile]:
# create toolboxposition element
sectionelement = ET.Element("toolboxposition")
sectionelement.attrib = attributes
sectionelement.tail = "\n "
newelements.append( ET.tostring(sectionelement, 'utf-8') )
if ( not hastags ):
# create empty tags element
newelements.append( "<tags/>\n " )
contents = (
contents[ 0:addelementsatposition ] +
newelements +
contents[ addelementsatposition: ] )
contents = ( contents[ 0:addelementsatposition ] + newelements +
contents[ addelementsatposition: ] )
# add .new for testing/safety purposes :P
newtoolconffile = open ( toolconffile, 'w' )
newtoolconffile = open( toolconffile, 'w' )
newtoolconffile.writelines( contents )
newtoolconffile.close()
@@ -104,9 +100,7 @@ def main():
def addToToolDict(tool, sectionname, sectionindex, toolindex, currentlabel):
toolfile = tool.attrib["file"]
realtoolfile = os.path.join(os.getcwd(), "tools", toolfile)
toolxmlfile = ET.parse(realtoolfile)
localroot = toolxmlfile.getroot()
# define attributes for the toolboxposition xml-tag
attribdict = {}
if ( sectionname ):
@@ -119,6 +113,7 @@ def addToToolDict(tool, sectionname, sectionindex, toolindex, currentlabel):
attribdict[ "order" ] = str(toolindex)
tooldict[ realtoolfile ].append(attribdict)
# Build a list of all toolconf xml files in the tools directory
def getfnl(startdir):
filenamelist = []
scripts/fetch_eggs.py
+6 -5
View File
@@ -5,7 +5,9 @@ If eggs for your platform are unavailable, fetch_eggs.py will direct you to run
scramble.py.
"""
import os, sys, logging
import logging
import os
import sys
from optparse import OptionParser
parser = OptionParser()
@@ -35,17 +37,16 @@ lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" )
sys.path.insert(1, lib)
from galaxy.eggs import Crate, EggNotFetchable
import pkg_resources
if options.platform:
c = Crate( config, platform = options.platform )
c = Crate( config, platform=options.platform )
else:
c = Crate( config )
try:
if not options.egg_name:
c.resolve() # Only fetch eggs required by the config
c.resolve() # Only fetch eggs required by the config
elif options.egg_name == 'all':
c.resolve( all=True ) # Fetch everything
c.resolve( all=True ) # Fetch everything
else:
# Fetch a specific egg
name = options.egg_name
scripts/functional_tests.py
+11 -23
View File
@@ -1,14 +1,10 @@
#!/usr/bin/env python
# Import system subprocess now before twill so we don't get its
# variant that breaks things.
import subprocess
import os
import sys
import shutil
import tempfile
import re
import shutil
import sys
import tempfile
from ConfigParser import SafeConfigParser
# Assume we are run from the galaxy root directory, add lib to the python path
@@ -25,7 +21,6 @@ from galaxy.util.properties import load_app_properties
eggs.require( "nose" )
eggs.require( "NoseHTML" )
eggs.require( "NoseTestDiff" )
eggs.require( "twill==0.9" )
eggs.require( "Paste" )
eggs.require( "PasteDeploy" )
eggs.require( "Cheetah" )
@@ -34,25 +29,18 @@ eggs.require( "Cheetah" )
# http://code.google.com/p/python-nose/issues/detail?id=284
eggs.require( "pysqlite" )
import atexit
import logging
import os.path
import twill
import unittest
import time
import subprocess
import threading
import random
import httplib
import socket
import urllib
from paste import httpserver
import galaxy.app
from galaxy.app import UniverseApplication
from galaxy.web import buildapp
from galaxy import tools
from galaxy.util import bunch
from galaxy import util
from galaxy.util.json import dumps
from functional import database_contexts
@@ -101,6 +89,7 @@ job_conf_xml = '''<?xml version="1.0"?>
</job_conf>
'''
def get_static_settings():
"""Returns dictionary of the settings necessary for a galaxy App
to be wrapped in the static middleware.
@@ -110,9 +99,9 @@ def get_static_settings():
"""
cwd = os.getcwd()
static_dir = os.path.join( cwd, 'static' )
#TODO: these should be copied from config/galaxy.ini
# TODO: these should be copied from config/galaxy.ini
return dict(
#TODO: static_enabled needed here?
# TODO: static_enabled needed here?
static_enabled=True,
static_cache_time=360,
static_dir=static_dir,
@@ -308,11 +297,11 @@ def main():
except OSError:
pass
#Data Manager testing temp path
#For storing Data Manager outputs and .loc files so that real ones don't get clobbered
# Data Manager testing temp path
# For storing Data Manager outputs and .loc files so that real ones don't get clobbered
data_manager_test_tmp_path = tempfile.mkdtemp( prefix='data_manager_test_tmp', dir=galaxy_test_tmp_dir )
galaxy_data_manager_data_path = tempfile.mkdtemp( prefix='data_manager_tool-data', dir=data_manager_test_tmp_path )
# ---- Build Application --------------------------------------------------
master_api_key = get_master_api_key()
app = None
@@ -348,8 +337,7 @@ def main():
use_tasked_jobs=True,
cleanup_job='onsuccess',
enable_beta_tool_formats=True,
data_manager_config_file=data_manager_config_file,
)
data_manager_config_file=data_manager_config_file )
if install_database_connection is not None:
kwargs[ 'install_database_connection' ] = install_database_connection
if not database_connection.startswith( 'sqlite://' ):
@@ -448,7 +436,7 @@ def main():
data_manager_test = __check_arg( '-data_managers', param=False )
if data_manager_test:
import functional.test_data_managers
functional.test_data_managers.data_managers = app.data_managers #seems like a hack...
functional.test_data_managers.data_managers = app.data_managers # seems like a hack...
functional.test_data_managers.build_tests(
tmp_dir=data_manager_test_tmp_path,
testing_shed_tools=testing_shed_tools,
scripts/get_platforms.py
+2 -2
View File
@@ -1,12 +1,12 @@
#!/usr/bin/env python
import os, sys
import os
import sys
assert sys.version_info[:2] >= ( 2, 4 )
lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) )
sys.path.insert( 1, lib )
import galaxy
import pkg_resources
print pkg_resources.get_platform()
scripts/helper.py
+7 -9
View File
@@ -5,7 +5,8 @@ Encodes and decodes IDs, returns Dataset IDs if provided an HDA or LDDA id,
returns the disk path of a dataset.
"""
import os, sys
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
@@ -28,19 +29,16 @@ except:
options.config = os.path.abspath( options.config )
sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) )
from galaxy import eggs
import pkg_resources
config = ConfigParser( dict( file_path = 'database/files',
id_secret = 'USING THE DEFAULT IS NOT SECURE!',
database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) )
config = ConfigParser( dict( file_path='database/files',
id_secret='USING THE DEFAULT IS NOT SECURE!',
database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) )
config.read( options.config )
from galaxy.web import security
from galaxy.model import mapping
helper = security.SecurityHelper( id_secret = config.get( 'app:main', 'id_secret' ) )
model = mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables = False )
helper = security.SecurityHelper( id_secret=config.get( 'app:main', 'id_secret' ) )
model = mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables=False )
if options.encode_id:
print 'Encoded "%s": %s' % ( options.encode_id, helper.encode_id( options.encode_id ) )
scripts/make_egg_packager.py
+12 -9
View File
@@ -1,8 +1,17 @@
#!/usr/bin/env python
import os, sys, logging, shutil
import logging
import os
import shutil
import sys
from optparse import OptionParser
lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) )
sys.path.insert( 1, lib )
from galaxy.eggs import Crate, py
import pkg_resources
parser = OptionParser()
parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default='config/galaxy.ini' )
parser.add_option( '-p', '--platform', dest='platform', help='Fetch for a specific platform (by default, eggs are fetched for *this* platform' )
@@ -16,16 +25,10 @@ root = logging.getLogger()
root.setLevel( 10 )
root.addHandler( logging.StreamHandler( sys.stdout ) )
lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) )
sys.path.insert( 1, lib )
from galaxy.eggs import Crate, EggNotFetchable, py
import pkg_resources
try:
assert options.platform
platform = options.platform
c = Crate( options.config, platform = platform )
c = Crate( options.config, platform=platform )
print "Platform forced to '%s'" % platform
except:
platform = '-'.join( ( py, pkg_resources.get_platform() ) )
@@ -63,7 +66,7 @@ if failures:
else:
create_zip()
clean()
""" )
""" )
print "Completed packager is 'egg_packager-%s.py'. To" % platform
print "fetch eggs, please copy this file to a system with internet access and run"
scripts/metagenomics/convert_title.py
+10 -9
View File
@@ -5,19 +5,21 @@ convert nt and wgs data (fasta format) to giNumber_seqLen
run formatdb in the command line: gunzip -c nt.gz |formatdb -i stdin -p F -n "nt.chunk" -v 2000
"""
import os, sys, math
import sys
if __name__ == '__main__':
seq = []
seq = []
len_seq = 0
invalid_lines = 0
gi = None
for i, line in enumerate(sys.stdin):
line = line.rstrip('\r\n')
if line.startswith('>'):
if len_seq > 0:
print ">%s_%d" %(gi, len_seq)
if gi is None:
raise Exception('The first sequence does not have an header.')
print ">%s_%d" % (gi, len_seq)
print "\n".join(seq)
title = line
fields = title.split('|')
@@ -32,8 +34,7 @@ if __name__ == '__main__':
seq.append(line)
len_seq += len(line)
if len_seq > 0:
print ">%s_%d" %(gi, len_seq)
print ">%s_%d" % (gi, len_seq)
print "\n".join(seq)
print >> sys.stderr, "Unable to find gi number for %d sequences, the title is replaced as giunknown" %(invalid_lines)
print >> sys.stderr, "Unable to find gi number for %d sequences, the title is replaced as giunknown" % (invalid_lines)
scripts/migrate_tools/migrate_tools.py
+2 -3
View File
@@ -2,7 +2,7 @@
This script will start up its own web application which includes a ToolMigrationManager (~/lib/galaxy/tool_shed/tool_migration_manager.py).
For each tool discovered missing, the tool shed repository that contains it will be installed on disk and a new entry will be
created for it in the migrated_tools_conf.xml file. These entries will be made so that the tool panel will be displayed the same
as it was before the tools were eliminated from the Galaxy distribution. The ToolMigrationManager will properly handle entries in
as it was before the tools were eliminated from the Galaxy distribution. The ToolMigrationManager will properly handle entries in
migrated_tools_conf.xml for tools outside tool panel sections as well as tools inside tool panel sections, depending upon the
layout of the local tool_conf.xml file. Entries will not be created in migrated_tools_conf.xml for tools included in the tool
shed repository but not defined in tool_conf.xml.
@@ -15,7 +15,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[ 1: ] )
sys.path = new_path
from galaxy import eggs
from tool_shed.galaxy_install.migrate.common import MigrateToolsApplication
app = MigrateToolsApplication( sys.argv[ 1 ] )
@@ -28,6 +27,6 @@ else:
file_names = ', '.join( non_shed_tool_confs )
msg = "\nThe installation process is finished. All tools associated with this migration that were defined in your file%s named\n" % plural
msg += "%s, have been removed. You may now start your Galaxy server.\n" % file_names
print msg
print msg
app.shutdown()
sys.exit( 0 )
scripts/nosetests.py
+4 -5
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python
# EASY-INSTALL-ENTRY-SCRIPT: 'nose','console_scripts','nosetests'
#__requires__ = 'nose'
import os, sys
# __requires__ = 'nose'
import os
import sys
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] )
@@ -15,6 +16,4 @@ from pkg_resources import load_entry_point
assert sys.version_info[:2] >= ( 2, 4 )
sys.exit(
load_entry_point('nose', 'console_scripts', 'nosetests')()
)
sys.exit( load_entry_point('nose', 'console_scripts', 'nosetests')() )
scripts/paster.py
+3 -8
View File
@@ -5,12 +5,8 @@ This should not be called directly! Use the run.sh script in Galaxy's
top level directly.
"""
import os, sys
try:
import configparser
except:
import ConfigParser as configparser
import os
import sys
# ensure supported version
from check_python import check_python
@@ -20,7 +16,7 @@ except:
sys.exit( 1 )
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
@@ -28,7 +24,6 @@ from galaxy import eggs
if 'LOG_TEMPFILES' in os.environ:
from log_tempfile import TempFile
_log_tempfile = TempFile()
import tempfile
eggs.require( "Paste" )
eggs.require( "PasteDeploy" )
scripts/runtime_stats.py
+5 -2
View File
@@ -31,9 +31,9 @@ Examples
"""
from __future__ import print_function
import argparse
import re
import sys
import argparse
try:
import configparser
@@ -130,6 +130,7 @@ def parse_arguments():
return args
def query(tool_id=None, user=None, like=None, source='metrics',
connect_args=None, debug=False, min=-1, max=-1, **kwargs):
@@ -248,8 +249,9 @@ def query(tool_id=None, user=None, like=None, source='metrics',
msg += ' (=%0.2f hours)' % hours
print(msg)
def nice_times(seconds):
if seconds < 60*60:
if seconds < 60 * 60:
hours = None
if seconds < 60:
minutes = None
@@ -260,6 +262,7 @@ def nice_times(seconds):
hours = seconds / 60 / 60
return hours, minutes
def main():
args = parse_arguments()
query(**vars(args))
scripts/scramble.py
+3 -1
View File
@@ -1,4 +1,6 @@
import os, sys, logging
import logging
import os
import sys
from optparse import OptionParser
parser = OptionParser()
scripts/set_dataset_sizes.py
+6 -8
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import os, sys
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
@@ -10,21 +11,18 @@ parser = OptionParser()
parser.add_option( '-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=default_config )
( options, args ) = parser.parse_args()
def init():
def init():
options.config = os.path.abspath( options.config )
sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) )
from galaxy import eggs
import pkg_resources
config = ConfigParser( dict( file_path = 'database/files',
database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) )
config = ConfigParser( dict( file_path='database/files',
database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) )
config.read( options.config )
from galaxy.model import mapping
return mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables = False )
return mapping.init( config.get( 'app:main', 'file_path' ), config.get( 'app:main', 'database_connection' ), create_tables=False )
if __name__ == '__main__':
print 'Loading Galaxy model...'
scripts/set_metadata.py
-2
View File
@@ -26,8 +26,6 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[ 1: ] ) # remove scripts/ from the path
sys.path = new_path
from galaxy import eggs
import pkg_resources
import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers
# This looks REAL stupid, but it is REQUIRED in order for SA to insert
scripts/set_user_disk_usage.py
+1 -6
View File
@@ -2,19 +2,15 @@
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) )
from galaxy import eggs
import pkg_resources
import galaxy.config
from galaxy.model.util import pgcalc
from galaxy.util import nice_size
from galaxy.objectstore import build_object_store_from_config
from galaxy.util import nice_size
default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'config/galaxy.ini') )
@@ -28,7 +24,6 @@ parser.add_option( '--dry-run', dest='dryrun', help='Dry run (show changes but d
def init():
options.config = os.path.abspath( options.config )
if options.username == 'all':
options.username = None
scripts/sync_reports_config.py
+5 -2
View File
@@ -4,6 +4,7 @@ from sys import argv
REPLACE_PROPERTIES = ["file_path", "database_connection", "new_file_path"]
MAIN_SECTION = "app:main"
def sync():
# Add or replace the relevant properites from galaxy.ini
# into reports_wsgi.ini
@@ -17,7 +18,7 @@ def sync():
parser = ConfigParser()
parser.read(universe_config_file)
with open(reports_config_file, "r") as f:
reports_config_lines = f.readlines()
@@ -38,6 +39,7 @@ def sync():
not (replacement_property in replaced_properties):
f.write(get_universe_line(replacement_property, parser))
def get_synced_line(reports_line, universe_config):
# Cycle through properties to replace and perform replacement on
# this line if needed.
@@ -51,8 +53,9 @@ def get_synced_line(reports_line, universe_config):
break
return (synced_line, replaced_property)
def get_universe_line(property_name, universe_config):
return "%s=%s\n" % (property_name, universe_config.get(MAIN_SECTION, property_name))
if __name__ == '__main__':
sync()
scripts/test_dist_egg.py
+5 -3
View File
@@ -6,7 +6,9 @@ for whatever egg you want to test.
usage: test_dist_egg.py <egg_name>
"""
import os, sys, logging, subprocess
import os
import subprocess
import sys
try:
assert sys.argv[1]
@@ -44,9 +46,9 @@ else:
name = sys.argv[1]
from galaxy.eggs.dist import DistScrambleCrate
c = DistScrambleCrate()
for egg in c[name]:
print 'Checking %s %s for %s on %s' % ( name, egg.version, egg.platform, egg.build_host )
p = subprocess.Popen( 'ssh %s %s %s %s %s' % ( egg.build_host, egg.python, os.path.abspath( __file__ ), egg.distribution.location, egg.platform ), shell=True )
scripts/update_shed_config_path.py
+9 -8
View File
@@ -6,13 +6,13 @@ new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] )
sys.path = new_path
import logging
import galaxy.model.tool_shed_install
import galaxy.model.tool_shed_install.mapping as mapping
from galaxy.model.orm import *
from galaxy import eggs
eggs.require('sqlalchemy')
import sqlalchemy
eggs.require('SQLAlchemy')
from sqlalchemy import create_engine, MetaData
from sqlalchemy.orm import scoped_session, sessionmaker
import galaxy.model.tool_shed_install.mapping as mapping
def main( opts, session, model ):
'''
@@ -26,6 +26,7 @@ def main( opts, session, model ):
session.flush()
return 0
def create_database( config_file ):
parser = ConfigParser.SafeConfigParser()
parser.read( config_file )
@@ -61,8 +62,8 @@ def create_database( config_file ):
# Initialize the database connection.
engine = create_engine( database_connection )
meta = MetaData( bind=engine )
install_session = Session = scoped_session( sessionmaker( bind=engine, autoflush=False, autocommit=True ) )
MetaData( bind=engine )
install_session = scoped_session( sessionmaker( bind=engine, autoflush=False, autocommit=True ) )
model = mapping.init( database_connection )
return install_session, model