From fbe21f94bf94cb75efb45f6dba95c3f5f6069aaf Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 11 Nov 2019 02:34:38 -0500 Subject: [PATCH] Remove tool dependency scripts. --- scripts/tool_shed/check_download_urls.py | 45 ----- ...mpty_tool_dependency_installation_paths.py | 72 -------- ...mpty_tool_dependency_installation_paths.py | 155 ------------------ .../clean_up_tool_dependency_directory.py | 34 ---- ...ol_dependency_installation_dir_contents.py | 75 --------- 5 files changed, 381 deletions(-) delete mode 100644 scripts/tool_shed/check_download_urls.py delete mode 100644 scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py delete mode 100644 scripts/tool_shed/check_s3_for_empty_tool_dependency_installation_paths.py delete mode 100644 scripts/tool_shed/clean_up_tool_dependency_directory.py delete mode 100644 scripts/tool_shed/show_tool_dependency_installation_dir_contents.py diff --git a/scripts/tool_shed/check_download_urls.py b/scripts/tool_shed/check_download_urls.py deleted file mode 100644 index 9664f0b8c8c..00000000000 --- a/scripts/tool_shed/check_download_urls.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# Dan Blankenberg -# Script that checks toolshed tags to see if URLs are accessible. -# Does not currently handle 'download_binary' -from __future__ import print_function - -import os -import xml.etree.ElementTree as ET -from optparse import OptionParser - -from six.moves.urllib.request import ( - Request, - urlopen -) - -FILENAMES = ['tool_dependencies.xml'] -ACTION_TYPES = ['download_by_url', 'download_file'] - - -def main(): - parser = OptionParser() - parser.add_option('-d', '--directory', dest='directory', action='store', type="string", default='.', help='Root directory') - - (options, args) = parser.parse_args() - - for (dirpath, dirnames, filenames) in os.walk(options.directory): - for filename in filenames: - if filename in FILENAMES: - path = os.path.join(dirpath, filename) - try: - tree = ET.parse(path) - root = tree.getroot() - for action_type in ACTION_TYPES: - for element in root.findall(".//action[@type='%s']" % action_type): - url = element.text.strip() - try: - urlopen(Request(url)) - except Exception as e: - print("Bad URL '%s' in file '%s': %s" % (url, path, e)) - except Exception as e: - print("Unable to check XML file '%s': %s" % (path, e)) - - -if __name__ == "__main__": - main() diff --git a/scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py b/scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py deleted file mode 100644 index eaab4633fda..00000000000 --- a/scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import print_function - -import argparse -import os -import shutil -import sys - -sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')) - -from tool_shed.util.basic_util import INSTALLATION_LOG - - -def main(args): - empty_installation_paths = [] - if not os.path.exists(args.basepath): - print('Tool dependency path "%s" does not exist.' % args.basepath) - return 1 - print('Checking path "%s" for empty tool dependency installation directories.' % args.basepath) - for root, dirs, files in os.walk(args.basepath): - path_parts = root.replace(args.basepath, '').lstrip('/').split(os.sep) - # Skip certain special directories. - if '__virtualenv_src' in dirs: - dirs.remove('__virtualenv_src') - if 'environment_settings' in dirs: - dirs.remove('environment_settings') - # Do not process the current path if it does not match the pattern - # ////. - if len(path_parts) != 5: - continue - # We have a tool dependency installation path. - no_dirs = False - no_files = False - if len(dirs) == 0: - no_dirs = True - if len(files) == 0 or len(files) == 1 and INSTALLATION_LOG in files: - no_files = True - if no_files and no_dirs and root not in empty_installation_paths: - empty_installation_paths.append(root) - if len(empty_installation_paths) > 0: - print('The following %d tool dependency installation directories were found to be empty or contain only the file %s.' % - (len(empty_installation_paths), INSTALLATION_LOG)) - if args.delete: - for path in empty_installation_paths: - if os.path.exists(path): - shutil.rmtree(path) - print('Deleted %s.' % path) - else: - for empty_installation_path in empty_installation_paths: - print(empty_installation_path) - else: - print('No empty tool dependency installation directories found.') - return 0 - - -if __name__ == '__main__': - description = 'Determine if there are any tool dependency installation paths that should be removed. Remove them if ' - description += 'the --delete command line argument is provided.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('--delete', - dest='delete', - required=False, - action='store_true', - default=False, - help='Whether to delete empty folders or list them on exit.') - parser.add_argument('--basepath', - dest='basepath', - required=True, - action='store', - metavar='name', - help='The base path where tool dependencies are installed.') - args = parser.parse_args() - sys.exit(main(args)) diff --git a/scripts/tool_shed/check_s3_for_empty_tool_dependency_installation_paths.py b/scripts/tool_shed/check_s3_for_empty_tool_dependency_installation_paths.py deleted file mode 100644 index 4005b29a0fd..00000000000 --- a/scripts/tool_shed/check_s3_for_empty_tool_dependency_installation_paths.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import print_function - -import argparse -import os -import sys - -sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')) - -import boto - -from galaxy.util import asbool -from tool_shed.util.basic_util import INSTALLATION_LOG - - -class BucketList(object): - - def __init__(self, amazon_id, amazon_secret, bucket): - # Connect to S3 using the provided Amazon access key and secret identifier. - self.s3 = boto.connect_s3(amazon_id, amazon_secret) - self.bucket_name = bucket - # Connect to S3 using the received bucket name. - self.bucket = boto.s3.bucket.Bucket(self.s3, bucket) - self.install_dirs = self.get_tool_dependency_install_paths() - self.empty_installation_paths = self.check_for_empty_tool_dependency_installation_paths() - - def display_empty_installation_paths(self): - for empty_installation_path in self.empty_installation_paths: - print(empty_installation_path) - - def delete_empty_installation_paths(self): - print('Deleting empty installation paths.') - for empty_installation_path in self.empty_installation_paths: - # Get all keys in the S3 bucket that start with the installation path, and delete each one. - for path_to_delete in self.bucket.list(prefix=empty_installation_path): - self.bucket.delete_key(path_to_delete.key) - print('Deleted empty path %s' % empty_installation_path) - - def get_tool_dependency_install_paths(self): - found_paths = [] - for item in self.bucket.list(): - name = str(item.name) - # Skip environment_settings and __virtualenv_src, since these directories do not contain package tool dependencies. - if name.startswith('environment_settings') or name.startswith('__virtualenv_src'): - continue - paths = name.rstrip('/').split('/') - # Paths are in the format name/version/owner/repository/changeset_revision. If the changeset revision is - # present, we need to check the contents of that path. If not, then the tool dependency was completely - # uninstalled. - if len(paths) >= 5: - td_install_dir = '/'.join(paths[:5]) + '/' - if td_install_dir not in found_paths: - found_paths.append(name) - return found_paths - - def check_for_empty_tool_dependency_installation_paths(self): - empty_directories = [] - for item in self.install_dirs: - # Get all entries under the path for this tool dependency. - contents = self.bucket.list(prefix=item) - tool_dependency_path_contents = [] - # Find out if there are two or less items in the path. The first entry will be the installation path itself. - # If only one other item exists, and the full path ends with the installation log, this is an incorrectly installed - # tool dependency. - for item in contents: - tool_dependency_path_contents.append(item) - # If there are more than two items in the path, we cannot safely assume that the dependency failed to - # install correctly. - if len(tool_dependency_path_contents) > 2: - break - # If the root directory is the only entry in the path, we have an empty tool dependency installation path. - if len(tool_dependency_path_contents) == 1: - empty_directories.append(tool_dependency_path_contents[0]) - # Otherwise, if the only other entry is the installation log, we have an installation path that should be deleted. - # This would not be the case in a Galaxy instance, since the Galaxy admin will need to verify the contents of - # the installation path in order to determine which action should be taken. - elif len(tool_dependency_path_contents) == 2 and \ - tool_dependency_path_contents[1].name.endswith(INSTALLATION_LOG): - empty_directories.append(tool_dependency_path_contents[0]) - return [item.name for item in empty_directories] - - -def main(args): - ''' - Amazon credentials can be provided in one of three ways: - 1. By specifying them on the command line with the --id and --secret arguments. - 2. By specifying a path to a file that contains the credentials in the form ACCESS_KEY:SECRET_KEY - using the --s3passwd argument. - 3. By specifying the above path in the 's3passwd' environment variable. - Each listed option will override the ones below it, if present. - ''' - if None in [args.id, args.secret]: - if args.s3passwd is None: - args.s3passwd = os.environ.get('s3passwd', None) - if args.s3passwd is not None and os.path.exists(args.s3passwd): - awsid, secret = open(args.s3passwd, 'r').read().rstrip('\n').split(':') - else: - print('Amazon ID and secret not provided, and no s3passwd file found.') - return 1 - else: - awsid = args.id - secret = args.secret - dependency_cleaner = BucketList(awsid, secret, args.bucket) - if len(dependency_cleaner.empty_installation_paths) == 0: - print('No empty installation paths found, exiting.') - return 0 - print('The following %d tool dependency installation paths were found to be empty or contain only the file %s.' % - (len(dependency_cleaner.empty_installation_paths), INSTALLATION_LOG)) - if asbool(args.delete): - dependency_cleaner.delete_empty_installation_paths() - else: - for empty_installation_path in dependency_cleaner.empty_installation_paths: - print(empty_installation_path) - return 0 - - -if __name__ == '__main__': - description = 'Determine if there are any tool dependency installation paths that should be removed. Remove them if ' - description += 'the --delete command line argument is provided with a true value.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('--delete', - dest='delete', - required=True, - action='store', - default=False, - type=asbool, - help='Whether to delete empty folders or list them on exit.') - parser.add_argument('--bucket', - dest='bucket', - required=True, - action='store', - metavar='name', - help='The S3 bucket where tool dependencies are installed.') - parser.add_argument('--id', - dest='id', - required=False, - action='store', - default=None, - metavar='ACCESS_KEY', - help='The identifier for an amazon account that has read access to the bucket.') - parser.add_argument('--secret', - dest='secret', - required=False, - action='store', - default=None, - metavar='SECRET_KEY', - help='The secret key for an amazon account that has upload/delete access to the bucket.') - parser.add_argument('--s3passwd', - dest='s3passwd', - required=False, - action='store', - default=None, - metavar='path/file', - help='The path to a file containing Amazon access credentials, in the format KEY:SECRET.') - args = parser.parse_args() - sys.exit(main(args)) diff --git a/scripts/tool_shed/clean_up_tool_dependency_directory.py b/scripts/tool_shed/clean_up_tool_dependency_directory.py deleted file mode 100644 index a8899c5f383..00000000000 --- a/scripts/tool_shed/clean_up_tool_dependency_directory.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import print_function - -import argparse -import os -import shutil -import sys - - -def main(args): - if not os.path.exists(args.tool_dependency_dir): - print('Tool dependency base path "%s" does not exist, creating.' % args.tool_dependency_dir) - os.mkdir(args.tool_dependency_dir) - return 0 - else: - for content in os.listdir(args.tool_dependency_dir): - print('Deleting directory "%s" from "%s" .' % (content, args.tool_dependency_dir)) - full_path = os.path.join(args.tool_dependency_dir, content) - if os.path.isdir(full_path): - shutil.rmtree(full_path) - else: - os.remove(full_path) - - -if __name__ == '__main__': - description = 'Clean out the configured tool dependency path, creating it if it does not exist.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('--tool_dependency_dir', - dest='tool_dependency_dir', - required=True, - action='store', - metavar='name', - help='The base path where tool dependencies will be installed.') - args = parser.parse_args() - sys.exit(main(args)) diff --git a/scripts/tool_shed/show_tool_dependency_installation_dir_contents.py b/scripts/tool_shed/show_tool_dependency_installation_dir_contents.py deleted file mode 100644 index b030a820c85..00000000000 --- a/scripts/tool_shed/show_tool_dependency_installation_dir_contents.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import print_function - -import argparse -import os -import sys - -sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')) - -import galaxy.config as galaxy_config -import galaxy.model -import galaxy.model.tool_shed_install.mapping as install_mapper - - -class CleanUpDependencyApplication(object): - """Application that enables querying the database using the tool_shed_install model.""" - - def __init__(self, config): - self.config = config - # Setup the database engine and ORM - self.model = install_mapper.init(self.config.database_connection, engine_options={}, create_tables=False) - - @property - def sa_session(self): - """Returns a SQLAlchemy session.""" - return self.model.context.current - - def shutdown(self): - pass - - -def main(args, app): - if not os.path.exists(args.basepath): - print('Tool dependency base path "%s" does not exist.' % args.basepath) - return - print('Checking tool dependency path %s' % args.basepath) - tool_dependency_dirs = get_tool_dependency_dirs(app) - for tool_dependency_dir in tool_dependency_dirs: - path = os.path.join(args.basepath, tool_dependency_dir) - if os.path.exists(path): - path_contents = os.listdir(path) - if len(path_contents) > 0: - print('Found non-empty tool dependency installation directory %s.' % path) - print('Directory has the following contents: \n %s' % '\n '.join(path_contents)) - - -def get_tool_dependency_dirs(app): - dependency_paths = [] - for tool_dependency in app.sa_session.query(galaxy.model.tool_shed_install.ToolDependency).all(): - dependency_paths.append(tool_dependency.installation_directory(app)) - return dependency_paths - - -if __name__ == '__main__': - description = 'Clean out or list the contents any tool dependency directory under the provided' - description += 'tool dependency path. Remove any non-empty directories found if the ' - description += '--delete command line argument is provided.' - parser = argparse.ArgumentParser(description=description) - parser.add_argument('--basepath', - dest='basepath', - required=True, - action='store', - metavar='name', - help='The base path where tool dependencies are installed.') - parser.add_argument('--dburi', - dest='dburi', - required=True, - action='store', - metavar='dburi', - help='The database URI to connect to.') - args = parser.parse_args() - database_connection = args.dburi - config_dict = dict(database_connection=database_connection, tool_dependency_dir=args.basepath) - config = galaxy_config.Configuration(**config_dict) - app = CleanUpDependencyApplication(config) - sys.exit(main(args, app))