opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 15:37:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Python3: scripts/cleanup_datasets scripts/data_libraries

Browse Source
This commit is contained in:
Junzhou Wang
2016-06-25 21:46:52 -04:00
parent 31c61caba3
commit f68a8198da
15 changed files with 108 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/py3_sources.txt
+14
View File
@@ -59,6 +59,20 @@ lib/tool_shed/util/
lib/tool_shed/utility_containers/
scripts/api/
scripts/auth/
scripts/bootstrap_history.py
scripts/build_toolbox.py
scripts/check_eggs.py
scripts/check_galaxy.py
scripts/check_python.py
scripts/cleanup_datasets/pgcleanup.py
scripts/cleanup_datasets/populate_uuid.py
scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py
scripts/cleanup_datasets/rename_purged_datasets.py
scripts/cleanup_datasets/update_dataset_size.py
scripts/cleanup_datasets/update_metadata.py
scripts/data_libraries/build_whoosh_index.py
scripts/db_shell.py
scripts/drmaa_external_runner.py
scripts/cleanup_datasets/admin_cleanup_datasets.py
scripts/cleanup_datasets/cleanup_datasets.py
test/api/test_workflows_from_yaml.py
scripts/bootstrap_history.py
+18 -18
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# Little script to make HISTORY.rst more easy to format properly, lots TODO
# pull message down and embed, use arg parse, handle multiple, etc...
from __future__ import print_function
import ast
import calendar
import datetime
@@ -13,14 +13,14 @@ try:
import requests
except ImportError:
requests = None
import urlparse
import textwrap
import json
try:
from pygithub3 import Github
except ImportError:
Github = None
from six.moves.urllib.parse import urljoin
from six import string_types
PROJECT_DIRECTORY = os.path.join(os.path.dirname(__file__), os.pardir)
SOURCE_DIR = os.path.join(PROJECT_DIRECTORY, "lib")
@@ -251,7 +251,7 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
def commit_time(commit_hash):
api_url = urlparse.urljoin(PROJECT_API, "commits/%s" % commit_hash)
api_url = urljoin(PROJECT_API, "commits/%s" % commit_hash)
req = requests.get(api_url).json()
return datetime.datetime.strptime(req["commit"]["committer"]["date"], "%Y-%m-%dT%H:%M:%SZ")
@@ -331,7 +331,7 @@ def check_blocking_prs(argv):
release_name = argv[2]
block = 0
for pr in _get_prs(release_name, state="open"):
print "WARN: Blocking PR| %s" % _pr_to_str(pr)
print("WARN: Blocking PR| %s" % _pr_to_str(pr))
block = 1
sys.exit(block)
@@ -349,20 +349,20 @@ def check_blocking_issues(argv):
for page in issues:
for issue in page:
if issue.milestone and issue.milestone.title == release_name and "Publication of Galaxy Release" not in issue.title:
print "WARN: Blocking issue| %s" % _issue_to_str(issue)
print("WARN: Blocking issue| %s" % _issue_to_str(issue))
block = 1
sys.exit(block)
def _pr_to_str(pr):
if isinstance(pr, basestring):
if isinstance(pr, string_types):
return pr
return "PR #%s (%s) %s" % (pr.number, pr.title, pr.html_url)
def _issue_to_str(pr):
if isinstance(pr, basestring):
if isinstance(pr, string_types):
return pr
return "Issue #%s (%s) %s" % (pr.number, pr.title, pr.html_url)
@@ -463,7 +463,7 @@ def main(argv):
if len(argv) > 2:
message = argv[2]
elif not (ident.startswith("pr") or ident.startswith("issue")):
api_url = urlparse.urljoin(PROJECT_API, "commits/%s" % ident)
api_url = urljoin(PROJECT_API, "commits/%s" % ident)
if req is None:
req = requests.get(api_url).json()
commit = req["commit"]
@@ -471,13 +471,13 @@ def main(argv):
message = get_first_sentence(message)
elif requests is not None and ident.startswith("pr"):
pull_request = ident[len("pr"):]
api_url = urlparse.urljoin(PROJECT_API, "pulls/%s" % pull_request)
api_url = urljoin(PROJECT_API, "pulls/%s" % pull_request)
if req is None:
req = requests.get(api_url).json()
message = req["title"]
elif requests is not None and ident.startswith("issue"):
issue = ident[len("issue"):]
api_url = urlparse.urljoin(PROJECT_API, "issues/%s" % issue)
api_url = urljoin(PROJECT_API, "issues/%s" % issue)
if req is None:
req = requests.get(api_url).json()
message = req["title"]
@@ -522,7 +522,7 @@ def main(argv):
def _text_target(github, pull_request):
labels = []
pr_number = None
if isinstance(pull_request, basestring):
if isinstance(pull_request, string_types):
pr_number = pull_request
else:
pr_number = pull_request.number
@@ -530,10 +530,10 @@ def _text_target(github, pull_request):
try:
labels = github.issues.labels.list_by_issue(int(pr_number), user=PROJECT_OWNER, repo=PROJECT_NAME)
except Exception as e:
print e
print(e)
is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False
if len(labels) == 0:
print 'No labels found for %s' % pr_number
print('No labels found for %s' % pr_number)
return None
for label in labels:
label_name = label.name.lower()
@@ -555,7 +555,7 @@ def _text_target(github, pull_request):
is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement
if not( is_bug or is_some_kind_of_enhancement or is_minor or is_merge ):
print "No kind/ or minor or merge label found for %s" % _pr_to_str(pull_request)
print("No kind/ or minor or merge label found for %s" % _pr_to_str(pull_request))
text_target = None
if is_minor or is_merge:
@@ -574,7 +574,7 @@ def _text_target(github, pull_request):
elif is_bug:
text_target = "bug"
else:
print "Logic problem, cannot determine section for %s" % _pr_to_str(pull_request)
print("Logic problem, cannot determine section for %s" % _pr_to_str(pull_request))
text_target = None
return text_target
@@ -597,8 +597,8 @@ def _latest_release():
def _releases():
all_files = sorted(os.listdir(RELEASES_PATH))
release_note_file_pattern = re.compile(r"\d+\.\d+.rst")
release_note_files = filter(lambda f: release_note_file_pattern.match(f), all_files)
return sorted(map(lambda f: f.rstrip('.rst'), release_note_files))
release_note_files = [f for f in all_files if release_note_file_pattern.match(f)]
return sorted([f.rstrip('.rst') for f in release_note_files])
def _get_major_version():
scripts/build_toolbox.py
+7 -6
View File
@@ -1,3 +1,4 @@
from __future__ import print_function
import os
from xml.etree import ElementTree as ET
@@ -21,7 +22,7 @@ def getfilenamelist(startdir):
try:
doc = ET.parse(fullfn)
except:
print "An OOPS on", fullfn
print("An OOPS on", fullfn)
raise
rootelement = doc.getroot()
# Only interpret those 'tool' XML files that have
@@ -30,7 +31,7 @@ def getfilenamelist(startdir):
if rootelement.findall('toolboxposition'):
filenamelist.append(fullfn)
else:
print "DBG> tool config does not have a <section>:", fullfn
print("DBG> tool config does not have a <section>:", fullfn)
return filenamelist
@@ -58,7 +59,7 @@ class ToolBox(object):
self.tools[("%05d-%s" % (sectionorder, section), label, order, section)].append(toolelement)
def addElementsTo(self, rootelement):
toolkeys = self.tools.keys()
toolkeys = list(self.tools.keys())
toolkeys.sort()
# Initialize the loop: IDs to zero, current section and label to ''
@@ -139,13 +140,13 @@ def scanfiles(filenamelist):
tagarray.append(tag.text)
attrib['tags'] = ",".join(tagarray)
else:
print "DBG> No tags in", fn
print("DBG> No tags in", fn)
# Build the tool element
newtoolelement = ET.Element('tool', attrib)
toolboxpositionelements = toolelement.findall('toolboxposition')
if not toolboxpositionelements:
print "DBG> %s has no toolboxposition" % fn
print("DBG> %s has no toolboxposition" % fn)
else:
for toolboxpositionelement in toolboxpositionelements:
toolbox.add(newtoolelement, toolboxpositionelement)
@@ -164,7 +165,7 @@ def assemble():
toolbox.addElementsTo(toolboxelement)
print prettify(toolboxelement)
print(prettify(toolboxelement))
if __name__ == "__main__":
assemble()
scripts/check_galaxy.py
+8 -7
View File
@@ -3,6 +3,7 @@
check_galaxy can be run by hand, although it is meant to run from cron
via the check_galaxy.sh script in Galaxy's cron/ directory.
"""
from __future__ import print_function
import filecmp
import formatter
import getopt
@@ -55,14 +56,14 @@ def usage():
try:
opts, args = getopt.getopt( sys.argv[1:], 'n' )
except getopt.GetoptError as e:
print str(e)
print(str(e))
usage()
if len( args ) < 1:
usage()
server = args[0]
if server.endswith(".g2.bx.psu.edu"):
if debug:
print "Checking a PSU Galaxy server, using maint file"
print("Checking a PSU Galaxy server, using maint file")
maint = "/errordocument/502/%s/maint" % args[0].split('.', 1)[0]
else:
maint = None
@@ -70,7 +71,7 @@ new_history = False
for o, a in opts:
if o == "-n":
if debug:
print "Specified -n, will create a new history"
print("Specified -n, will create a new history")
new_history = True
else:
usage()
@@ -78,7 +79,7 @@ for o, a in opts:
# state information
var_dir = os.path.join( os.path.expanduser('~'), ".check_galaxy", server )
if not os.access( var_dir, os.F_OK ):
os.makedirs( var_dir, 0700 )
os.makedirs( var_dir, 0o700 )
# get user/pass
login_file = os.path.join( var_dir, "login" )
@@ -137,7 +138,7 @@ class Browser:
p = didParser()
p.feed(tc.browser.get_html())
if len(p.dids) > 0:
print "Remaining datasets ids:", " ".join( p.dids )
print("Remaining datasets ids:", " ".join( p.dids ))
raise Exception("History still contains datasets after attempting to delete them")
if new_history:
self.get("/history/delete_current")
@@ -363,7 +364,7 @@ class loggedinParser(htmllib.HTMLParser):
def dprint(str):
if debug:
print str
print(str)
# do stuff here
if __name__ == "__main__":
@@ -379,7 +380,7 @@ if __name__ == "__main__":
dprint("not logged in... logging in")
b.login(username, password)
for tool, params in tools.iteritems():
for tool, params in tools.items():
check_file = ""
scripts/check_python.py
+3 -3
View File
@@ -2,7 +2,7 @@
If the current installed python version is not 2.7, prints an error
message to stderr and returns 1
"""
from __future__ import print_function
import sys
msg = """ERROR: Your Python version is: %s
@@ -16,13 +16,13 @@ def check_python():
try:
assert sys.version_info[:2] == ( 2, 7 )
except AssertionError:
print >>sys.stderr, msg
print(msg, file=sys.stderr)
raise
if __name__ == '__main__':
rval = 0
try:
check_python()
except StandardError:
except Exception:
rval = 1
sys.exit( rval )
scripts/cleanup_datasets/pgcleanup.py
+3 -3
View File
@@ -4,14 +4,14 @@ pgcleanup.py - A script for cleaning up datasets in Galaxy efficiently, by
bypassing the Galaxy model and operating directly on the database.
PostgreSQL 9.1 or greater is required.
"""
from __future__ import print_function
import datetime
import inspect
import logging
import os
import shutil
import sys
from ConfigParser import ConfigParser
from configparser import ConfigParser
from optparse import OptionParser
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
@@ -76,7 +76,7 @@ class Cleanup(object):
self.options.sequence = [ x.strip() for x in self.options.sequence.split(',') ]
if self.options.sequence == ['']:
print "Error: At least one action must be specified in the action sequence\n"
print("Error: At least one action must be specified in the action sequence\n")
parser.print_help()
sys.exit(0)
scripts/cleanup_datasets/populate_uuid.py
+6 -6
View File
@@ -6,7 +6,7 @@ Populates blank uuid fields in datasets with randomly generated values
Going forward, these ids will be generated for all new datasets. This
script fixes datasets that were generated before the change.
"""
from __future__ import print_function
import sys
import uuid
@@ -17,13 +17,13 @@ assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog):
print "usage: %s galaxy.ini" % prog
print """
print("usage: %s galaxy.ini" % prog)
print("""
Populates blank uuid fields in datasets with randomly generated values.
Going forward, these ids will be generated for all new datasets. This
script fixes datasets that were generated before the change.
"""
""")
def main():
@@ -38,13 +38,13 @@ def main():
for row in model.context.query( model.Dataset ):
if row.uuid is None:
row.uuid = uuid.uuid4()
print "Setting dataset:", row.id, " UUID to ", row.uuid
print("Setting dataset:", row.id, " UUID to ", row.uuid)
model.context.flush()
for row in model.context.query( model.Workflow ):
if row.uuid is None:
row.uuid = uuid.uuid4()
print "Setting Workflow:", row.id, " UUID to ", row.uuid
print("Setting Workflow:", row.id, " UUID to ", row.uuid)
model.context.flush()
scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py
+8 -8
View File
@@ -3,7 +3,7 @@
Removes a dataset file ( which was first renamed by appending _purged to the file name ) from disk.
Usage: python remove_renamed_datasets_from_disk.py renamed.log
"""
from __future__ import print_function
import os
import sys
@@ -11,15 +11,15 @@ assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog):
print "usage: %s file" % prog
print """
print("usage: %s file" % prog)
print("""
Removes a set of files from disk. The input file should contain a list of files
to be deleted, one per line. The full path must be specified and must begin
with /var/opt/galaxy.
A log of files deleted is created in a file with the same name as that input but
with .removed.log appended.
"""
""")
def main():
@@ -30,7 +30,7 @@ def main():
outfile = infile + ".removed.log"
out = open( outfile, 'w' )
print >> out, "# The following renamed datasets have been removed from disk"
print("# The following renamed datasets have been removed from disk", file=out)
i = 0
removed_files = 0
for i, line in enumerate( open( infile ) ):
@@ -38,11 +38,11 @@ def main():
if line and line.startswith( '/var/opt/galaxy' ):
try:
os.unlink( line )
print >> out, line
print(line, file=out)
removed_files += 1
except Exception as exc:
print >> out, "# Error, exception " + str( exc ) + " caught attempting to remove " + line
print >> out, "# Removed " + str( removed_files ) + " files"
print("# Error, exception " + str( exc ) + " caught attempting to remove " + line, file=out)
print("# Removed " + str( removed_files ) + " files", file=out)
if __name__ == "__main__":
scripts/cleanup_datasets/rename_purged_datasets.py
+8 -8
View File
@@ -3,7 +3,7 @@
Renames a dataset file by appending _purged to the file name so that it can later be removed from disk.
Usage: python rename_purged_datasets.py purge.log
"""
from __future__ import print_function
import os
import sys
@@ -11,8 +11,8 @@ assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog):
print "usage: %s file" % prog
print """
print("usage: %s file" % prog)
print("""
Marks a set of files as purged and renames them. The input file should contain a
list of files to be purged, one per line. The full path must be specified and
must begin with /var/opt/galaxy.
@@ -20,7 +20,7 @@ A log of files marked as purged is created in a file with the same name as that
input but with _purged appended. The resulting files can finally be removed from
disk with remove_renamed_datasets_from_disk.py, by supplying it with a list of
them.
"""
""")
def main():
@@ -31,7 +31,7 @@ def main():
outfile = infile + ".renamed.log"
out = open( outfile, 'w' )
print >> out, "# The following renamed datasets can be removed from disk"
print("# The following renamed datasets can be removed from disk", file=out)
i = 0
renamed_files = 0
for i, line in enumerate( open( infile ) ):
@@ -40,11 +40,11 @@ def main():
try:
purged_filename = line + "_purged"
os.rename( line, purged_filename )
print >> out, purged_filename
print(purged_filename, file=out)
renamed_files += 1
except Exception as exc:
print >> out, "# Error, exception " + str( exc ) + " caught attempting to rename " + purged_filename
print >> out, "# Renamed " + str( renamed_files ) + " files"
print("# Error, exception " + str( exc ) + " caught attempting to rename " + purged_filename, file=out)
print("# Renamed " + str( renamed_files ) + " files", file=out)
if __name__ == "__main__":
main()
scripts/cleanup_datasets/update_dataset_size.py
+8 -8
View File
@@ -3,22 +3,22 @@
Updates dataset.size column.
Remember to backup your database before running.
"""
import ConfigParser
from __future__ import print_function
import os
import sys
import galaxy.app
from six.moves import configparser
assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog):
print "usage: %s galaxy.ini" % prog
print """
print("usage: %s galaxy.ini" % prog)
print("""
Updates the dataset.size column. Users are advised to backup the database before
running.
"""
""")
def main():
@@ -26,7 +26,7 @@ def main():
usage(sys.argv[0])
sys.exit()
ini_file = sys.argv.pop(1)
conf_parser = ConfigParser.ConfigParser( {'here': os.getcwd()} )
conf_parser = configparser.ConfigParser( {'here': os.getcwd()} )
conf_parser.read( ini_file )
configuration = {}
for key, value in conf_parser.items( "app:main" ):
@@ -34,13 +34,13 @@ def main():
app = galaxy.app.UniverseApplication( global_conf=ini_file, **configuration )
# Step through Datasets, determining size on disk for each.
print "Determining the size of each dataset..."
print("Determining the size of each dataset...")
for row in app.model.Dataset.table.select().execute():
purged = app.model.Dataset.get( row.id ).purged
file_size = app.model.Dataset.get( row.id ).file_size
if file_size is None and not purged:
size_on_disk = app.model.Dataset.get( row.id ).get_size()
print "Updating Dataset.%d with file_size: %d" % ( row.id, size_on_disk )
print("Updating Dataset.%d with file_size: %d" % ( row.id, size_on_disk ))
app.model.Dataset.table.update( app.model.Dataset.table.c.id == row.id ).execute( file_size=size_on_disk )
app.shutdown()
sys.exit(0)
scripts/cleanup_datasets/update_metadata.py
+8 -8
View File
@@ -5,10 +5,10 @@ Updates metadata in the database to match rev 1891.
Remember to backup your database before running.
"""
import ConfigParser
from __future__ import print_function
import os
import sys
from six.moves import configparser
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
@@ -19,12 +19,12 @@ assert sys.version_info[:2] >= ( 2, 4 )
def usage(prog):
print "usage: %s galaxy.ini" % prog
print """
print("usage: %s galaxy.ini" % prog)
print("""
Updates the metadata in the database to match rev 1981.
Remember to backup your database before running.
"""
""")
def main():
@@ -32,7 +32,7 @@ def main():
usage(sys.argv[0])
sys.exit()
ini_file = sys.argv.pop(1)
conf_parser = ConfigParser.ConfigParser({'here': os.getcwd()})
conf_parser = configparser.ConfigParser({'here': os.getcwd()})
conf_parser.read(ini_file)
configuration = {}
for key, value in conf_parser.items("app:main"):
@@ -40,11 +40,11 @@ def main():
app = galaxy.app.UniverseApplication( global_conf=ini_file, **configuration )
# Search out tabular datatypes (and subclasses) and initialize metadata
print "Seeking out tabular based files and initializing metadata"
print("Seeking out tabular based files and initializing metadata")
for row in app.model.Dataset.table.select().execute():
data = app.model.Dataset.get(row.id)
if issubclass(type(data.datatype), type(app.datatypes_registry.get_datatype_by_extension('tabular'))):
print row.id, data.extension
print(row.id, data.extension)
# Call meta_data for all tabular files
# special case interval type where we do not want to overwrite chr, start, end, etc assignments
if issubclass(type(data.datatype), type(app.datatypes_registry.get_datatype_by_extension('interval'))):
scripts/data_libraries/build_whoosh_index.py
+5 -4
View File
@@ -8,7 +8,8 @@ data library search section for more details.
Run from the ~/scripts/data_libraries directory:
%sh build_whoosh_index.sh
"""
import ConfigParser
from __future__ import print_function
from six.moves import configparser
import os
import sys
@@ -34,7 +35,7 @@ def build_index( sa_session, whoosh_index_dir ):
def to_unicode( a_basestr ):
if type( a_basestr ) is str:
return unicode( a_basestr, 'utf-8' )
return str( a_basestr, 'utf-8' )
else:
return a_basestr
lddas_indexed = 0
@@ -46,7 +47,7 @@ def build_index( sa_session, whoosh_index_dir ):
message=to_unicode( message ) )
lddas_indexed += 1
writer.commit()
print "Number of active library datasets indexed: ", lddas_indexed
print("Number of active library datasets indexed: ", lddas_indexed)
def get_lddas( sa_session ):
@@ -67,7 +68,7 @@ def get_lddas( sa_session ):
def get_sa_session_and_needed_config_settings( ini_file ):
conf_parser = ConfigParser.ConfigParser( { 'here': os.getcwd() } )
conf_parser = configparser.ConfigParser( { 'here': os.getcwd() } )
conf_parser.read( ini_file )
kwds = dict()
for key, value in conf_parser.items( "app:main" ):
scripts/db_shell.py
+7 -2
View File
@@ -10,10 +10,15 @@
# You can also use this script as a library, for instance see https://gist.github.com/1979583
# TODO: This script overlaps a lot with manage_db.py and create_db.py,
# these should maybe be refactored to remove duplication.
from __future__ import print_function
import datetime
import decimal
import os.path
import sys
from six import string_types, PY3
if PY3:
long = int
# Setup DB scripting environment
from sqlalchemy import * # noqa
@@ -73,7 +78,7 @@ def printquery(statement, bind=None):
of the DBAPI.
"""
if isinstance(value, basestring):
if isinstance(value, string_types):
value = value.replace("'", "''")
return "'%s'" % value
elif value is None:
@@ -91,4 +96,4 @@ def printquery(statement, bind=None):
)
compiler = LiteralCompiler(dialect, statement)
print compiler.process(statement)
print(compiler.process(statement))
scripts/drmaa_external_runner.py
+2 -2
View File
@@ -5,7 +5,7 @@ Submit a DRMAA job given a user id and a job template file (in JSON format)
defining any or all of the following: args, remoteCommand, outputPath,
errorPath, nativeSpecification, name, email, project
"""
from __future__ import print_function
import errno
import json
import os
@@ -129,7 +129,7 @@ def main():
s.exit()
# Print the Job-ID and exit. Galaxy will pick it up from there.
print jobId
print(jobId)
if __name__ == "__main__":
main()
scripts/fetch_eggs.py
+3 -3
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
from os import pardir
from os.path import join, abspath, dirname
from sys import exit
@@ -27,6 +27,6 @@ cd {dir} && ./scripts/common_startup.sh --skip-venv
galaxy = abspath(join(dirname(__file__), pardir))
venv = join(galaxy, '.venv')
print msg.format(dir=abspath(join(dirname(__file__), pardir)),
venv=venv)
print(msg.format(dir=abspath(join(dirname(__file__), pardir)),
venv=venv))
exit(1)