mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Remove tool dependency scripts.
This commit is contained in:
John Chilton
@@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Dan Blankenberg
|
||||
# Script that checks toolshed tags to see if URLs are accessible.
|
||||
# Does not currently handle 'download_binary'
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import xml.etree.ElementTree as ET
|
||||
from optparse import OptionParser
|
||||
|
||||
from six.moves.urllib.request import (
|
||||
Request,
|
||||
urlopen
|
||||
)
|
||||
|
||||
FILENAMES = ['tool_dependencies.xml']
|
||||
ACTION_TYPES = ['download_by_url', 'download_file']
|
||||
|
||||
|
||||
def main():
|
||||
parser = OptionParser()
|
||||
parser.add_option('-d', '--directory', dest='directory', action='store', type="string", default='.', help='Root directory')
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
for (dirpath, dirnames, filenames) in os.walk(options.directory):
|
||||
for filename in filenames:
|
||||
if filename in FILENAMES:
|
||||
path = os.path.join(dirpath, filename)
|
||||
try:
|
||||
tree = ET.parse(path)
|
||||
root = tree.getroot()
|
||||
for action_type in ACTION_TYPES:
|
||||
for element in root.findall(".//action[@type='%s']" % action_type):
|
||||
url = element.text.strip()
|
||||
try:
|
||||
urlopen(Request(url))
|
||||
except Exception as e:
|
||||
print("Bad URL '%s' in file '%s': %s" % (url, path, e))
|
||||
except Exception as e:
|
||||
print("Unable to check XML file '%s': %s" % (path, e))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,72 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))
|
||||
|
||||
from tool_shed.util.basic_util import INSTALLATION_LOG
|
||||
|
||||
|
||||
def main(args):
|
||||
empty_installation_paths = []
|
||||
if not os.path.exists(args.basepath):
|
||||
print('Tool dependency path "%s" does not exist.' % args.basepath)
|
||||
return 1
|
||||
print('Checking path "%s" for empty tool dependency installation directories.' % args.basepath)
|
||||
for root, dirs, files in os.walk(args.basepath):
|
||||
path_parts = root.replace(args.basepath, '').lstrip('/').split(os.sep)
|
||||
# Skip certain special directories.
|
||||
if '__virtualenv_src' in dirs:
|
||||
dirs.remove('__virtualenv_src')
|
||||
if 'environment_settings' in dirs:
|
||||
dirs.remove('environment_settings')
|
||||
# Do not process the current path if it does not match the pattern
|
||||
# <name>/<version>/<owner>/<repository>/<changeset>.
|
||||
if len(path_parts) != 5:
|
||||
continue
|
||||
# We have a tool dependency installation path.
|
||||
no_dirs = False
|
||||
no_files = False
|
||||
if len(dirs) == 0:
|
||||
no_dirs = True
|
||||
if len(files) == 0 or len(files) == 1 and INSTALLATION_LOG in files:
|
||||
no_files = True
|
||||
if no_files and no_dirs and root not in empty_installation_paths:
|
||||
empty_installation_paths.append(root)
|
||||
if len(empty_installation_paths) > 0:
|
||||
print('The following %d tool dependency installation directories were found to be empty or contain only the file %s.' %
|
||||
(len(empty_installation_paths), INSTALLATION_LOG))
|
||||
if args.delete:
|
||||
for path in empty_installation_paths:
|
||||
if os.path.exists(path):
|
||||
shutil.rmtree(path)
|
||||
print('Deleted %s.' % path)
|
||||
else:
|
||||
for empty_installation_path in empty_installation_paths:
|
||||
print(empty_installation_path)
|
||||
else:
|
||||
print('No empty tool dependency installation directories found.')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
description = 'Determine if there are any tool dependency installation paths that should be removed. Remove them if '
|
||||
description += 'the --delete command line argument is provided.'
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--delete',
|
||||
dest='delete',
|
||||
required=False,
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Whether to delete empty folders or list them on exit.')
|
||||
parser.add_argument('--basepath',
|
||||
dest='basepath',
|
||||
required=True,
|
||||
action='store',
|
||||
metavar='name',
|
||||
help='The base path where tool dependencies are installed.')
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(args))
|
||||
@@ -1,155 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))
|
||||
|
||||
import boto
|
||||
|
||||
from galaxy.util import asbool
|
||||
from tool_shed.util.basic_util import INSTALLATION_LOG
|
||||
|
||||
|
||||
class BucketList(object):
|
||||
|
||||
def __init__(self, amazon_id, amazon_secret, bucket):
|
||||
# Connect to S3 using the provided Amazon access key and secret identifier.
|
||||
self.s3 = boto.connect_s3(amazon_id, amazon_secret)
|
||||
self.bucket_name = bucket
|
||||
# Connect to S3 using the received bucket name.
|
||||
self.bucket = boto.s3.bucket.Bucket(self.s3, bucket)
|
||||
self.install_dirs = self.get_tool_dependency_install_paths()
|
||||
self.empty_installation_paths = self.check_for_empty_tool_dependency_installation_paths()
|
||||
|
||||
def display_empty_installation_paths(self):
|
||||
for empty_installation_path in self.empty_installation_paths:
|
||||
print(empty_installation_path)
|
||||
|
||||
def delete_empty_installation_paths(self):
|
||||
print('Deleting empty installation paths.')
|
||||
for empty_installation_path in self.empty_installation_paths:
|
||||
# Get all keys in the S3 bucket that start with the installation path, and delete each one.
|
||||
for path_to_delete in self.bucket.list(prefix=empty_installation_path):
|
||||
self.bucket.delete_key(path_to_delete.key)
|
||||
print('Deleted empty path %s' % empty_installation_path)
|
||||
|
||||
def get_tool_dependency_install_paths(self):
|
||||
found_paths = []
|
||||
for item in self.bucket.list():
|
||||
name = str(item.name)
|
||||
# Skip environment_settings and __virtualenv_src, since these directories do not contain package tool dependencies.
|
||||
if name.startswith('environment_settings') or name.startswith('__virtualenv_src'):
|
||||
continue
|
||||
paths = name.rstrip('/').split('/')
|
||||
# Paths are in the format name/version/owner/repository/changeset_revision. If the changeset revision is
|
||||
# present, we need to check the contents of that path. If not, then the tool dependency was completely
|
||||
# uninstalled.
|
||||
if len(paths) >= 5:
|
||||
td_install_dir = '/'.join(paths[:5]) + '/'
|
||||
if td_install_dir not in found_paths:
|
||||
found_paths.append(name)
|
||||
return found_paths
|
||||
|
||||
def check_for_empty_tool_dependency_installation_paths(self):
|
||||
empty_directories = []
|
||||
for item in self.install_dirs:
|
||||
# Get all entries under the path for this tool dependency.
|
||||
contents = self.bucket.list(prefix=item)
|
||||
tool_dependency_path_contents = []
|
||||
# Find out if there are two or less items in the path. The first entry will be the installation path itself.
|
||||
# If only one other item exists, and the full path ends with the installation log, this is an incorrectly installed
|
||||
# tool dependency.
|
||||
for item in contents:
|
||||
tool_dependency_path_contents.append(item)
|
||||
# If there are more than two items in the path, we cannot safely assume that the dependency failed to
|
||||
# install correctly.
|
||||
if len(tool_dependency_path_contents) > 2:
|
||||
break
|
||||
# If the root directory is the only entry in the path, we have an empty tool dependency installation path.
|
||||
if len(tool_dependency_path_contents) == 1:
|
||||
empty_directories.append(tool_dependency_path_contents[0])
|
||||
# Otherwise, if the only other entry is the installation log, we have an installation path that should be deleted.
|
||||
# This would not be the case in a Galaxy instance, since the Galaxy admin will need to verify the contents of
|
||||
# the installation path in order to determine which action should be taken.
|
||||
elif len(tool_dependency_path_contents) == 2 and \
|
||||
tool_dependency_path_contents[1].name.endswith(INSTALLATION_LOG):
|
||||
empty_directories.append(tool_dependency_path_contents[0])
|
||||
return [item.name for item in empty_directories]
|
||||
|
||||
|
||||
def main(args):
|
||||
'''
|
||||
Amazon credentials can be provided in one of three ways:
|
||||
1. By specifying them on the command line with the --id and --secret arguments.
|
||||
2. By specifying a path to a file that contains the credentials in the form ACCESS_KEY:SECRET_KEY
|
||||
using the --s3passwd argument.
|
||||
3. By specifying the above path in the 's3passwd' environment variable.
|
||||
Each listed option will override the ones below it, if present.
|
||||
'''
|
||||
if None in [args.id, args.secret]:
|
||||
if args.s3passwd is None:
|
||||
args.s3passwd = os.environ.get('s3passwd', None)
|
||||
if args.s3passwd is not None and os.path.exists(args.s3passwd):
|
||||
awsid, secret = open(args.s3passwd, 'r').read().rstrip('\n').split(':')
|
||||
else:
|
||||
print('Amazon ID and secret not provided, and no s3passwd file found.')
|
||||
return 1
|
||||
else:
|
||||
awsid = args.id
|
||||
secret = args.secret
|
||||
dependency_cleaner = BucketList(awsid, secret, args.bucket)
|
||||
if len(dependency_cleaner.empty_installation_paths) == 0:
|
||||
print('No empty installation paths found, exiting.')
|
||||
return 0
|
||||
print('The following %d tool dependency installation paths were found to be empty or contain only the file %s.' %
|
||||
(len(dependency_cleaner.empty_installation_paths), INSTALLATION_LOG))
|
||||
if asbool(args.delete):
|
||||
dependency_cleaner.delete_empty_installation_paths()
|
||||
else:
|
||||
for empty_installation_path in dependency_cleaner.empty_installation_paths:
|
||||
print(empty_installation_path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
description = 'Determine if there are any tool dependency installation paths that should be removed. Remove them if '
|
||||
description += 'the --delete command line argument is provided with a true value.'
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--delete',
|
||||
dest='delete',
|
||||
required=True,
|
||||
action='store',
|
||||
default=False,
|
||||
type=asbool,
|
||||
help='Whether to delete empty folders or list them on exit.')
|
||||
parser.add_argument('--bucket',
|
||||
dest='bucket',
|
||||
required=True,
|
||||
action='store',
|
||||
metavar='name',
|
||||
help='The S3 bucket where tool dependencies are installed.')
|
||||
parser.add_argument('--id',
|
||||
dest='id',
|
||||
required=False,
|
||||
action='store',
|
||||
default=None,
|
||||
metavar='ACCESS_KEY',
|
||||
help='The identifier for an amazon account that has read access to the bucket.')
|
||||
parser.add_argument('--secret',
|
||||
dest='secret',
|
||||
required=False,
|
||||
action='store',
|
||||
default=None,
|
||||
metavar='SECRET_KEY',
|
||||
help='The secret key for an amazon account that has upload/delete access to the bucket.')
|
||||
parser.add_argument('--s3passwd',
|
||||
dest='s3passwd',
|
||||
required=False,
|
||||
action='store',
|
||||
default=None,
|
||||
metavar='path/file',
|
||||
help='The path to a file containing Amazon access credentials, in the format KEY:SECRET.')
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(args))
|
||||
@@ -1,34 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
|
||||
def main(args):
|
||||
if not os.path.exists(args.tool_dependency_dir):
|
||||
print('Tool dependency base path "%s" does not exist, creating.' % args.tool_dependency_dir)
|
||||
os.mkdir(args.tool_dependency_dir)
|
||||
return 0
|
||||
else:
|
||||
for content in os.listdir(args.tool_dependency_dir):
|
||||
print('Deleting directory "%s" from "%s" .' % (content, args.tool_dependency_dir))
|
||||
full_path = os.path.join(args.tool_dependency_dir, content)
|
||||
if os.path.isdir(full_path):
|
||||
shutil.rmtree(full_path)
|
||||
else:
|
||||
os.remove(full_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
description = 'Clean out the configured tool dependency path, creating it if it does not exist.'
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--tool_dependency_dir',
|
||||
dest='tool_dependency_dir',
|
||||
required=True,
|
||||
action='store',
|
||||
metavar='name',
|
||||
help='The base path where tool dependencies will be installed.')
|
||||
args = parser.parse_args()
|
||||
sys.exit(main(args))
|
||||
@@ -1,75 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))
|
||||
|
||||
import galaxy.config as galaxy_config
|
||||
import galaxy.model
|
||||
import galaxy.model.tool_shed_install.mapping as install_mapper
|
||||
|
||||
|
||||
class CleanUpDependencyApplication(object):
|
||||
"""Application that enables querying the database using the tool_shed_install model."""
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
# Setup the database engine and ORM
|
||||
self.model = install_mapper.init(self.config.database_connection, engine_options={}, create_tables=False)
|
||||
|
||||
@property
|
||||
def sa_session(self):
|
||||
"""Returns a SQLAlchemy session."""
|
||||
return self.model.context.current
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
|
||||
def main(args, app):
|
||||
if not os.path.exists(args.basepath):
|
||||
print('Tool dependency base path "%s" does not exist.' % args.basepath)
|
||||
return
|
||||
print('Checking tool dependency path %s' % args.basepath)
|
||||
tool_dependency_dirs = get_tool_dependency_dirs(app)
|
||||
for tool_dependency_dir in tool_dependency_dirs:
|
||||
path = os.path.join(args.basepath, tool_dependency_dir)
|
||||
if os.path.exists(path):
|
||||
path_contents = os.listdir(path)
|
||||
if len(path_contents) > 0:
|
||||
print('Found non-empty tool dependency installation directory %s.' % path)
|
||||
print('Directory has the following contents: \n %s' % '\n '.join(path_contents))
|
||||
|
||||
|
||||
def get_tool_dependency_dirs(app):
|
||||
dependency_paths = []
|
||||
for tool_dependency in app.sa_session.query(galaxy.model.tool_shed_install.ToolDependency).all():
|
||||
dependency_paths.append(tool_dependency.installation_directory(app))
|
||||
return dependency_paths
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
description = 'Clean out or list the contents any tool dependency directory under the provided'
|
||||
description += 'tool dependency path. Remove any non-empty directories found if the '
|
||||
description += '--delete command line argument is provided.'
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument('--basepath',
|
||||
dest='basepath',
|
||||
required=True,
|
||||
action='store',
|
||||
metavar='name',
|
||||
help='The base path where tool dependencies are installed.')
|
||||
parser.add_argument('--dburi',
|
||||
dest='dburi',
|
||||
required=True,
|
||||
action='store',
|
||||
metavar='dburi',
|
||||
help='The database URI to connect to.')
|
||||
args = parser.parse_args()
|
||||
database_connection = args.dburi
|
||||
config_dict = dict(database_connection=database_connection, tool_dependency_dir=args.basepath)
|
||||
config = galaxy_config.Configuration(**config_dict)
|
||||
app = CleanUpDependencyApplication(config)
|
||||
sys.exit(main(args, app))
|
||||
Reference in New Issue
Block a user