Merge pull request #5769 from jmchilton/unify_scripts

[18.01] Fix scripts/ for YAML/ENV configuration processing

lib/galaxy/config.py

@@ -860,6 +860,31 @@ def get_database_engine_options(kwargs, model_prefix=''):
     return rval
 
 
+def get_database_url(config):
+    if config.database_connection:
+        db_url = config.database_connection
+    else:
+        db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database
+    return db_url
+
+
+def init_models_from_config(config, map_install_models=False, object_store=None, trace_logger=None):
+    db_url = get_database_url(config)
+    from galaxy.model import mapping
+    model = mapping.init(
+        config.file_path,
+        db_url,
+        config.database_engine_options,
+        map_install_models=map_install_models,
+        database_query_profiling_proxy=config.database_query_profiling_proxy,
+        object_store=object_store,
+        trace_logger=trace_logger,
+        use_pbkdf2=config.get_bool('use_pbkdf2', True),
+        slow_query_log_threshold=config.slow_query_log_threshold
+    )
+    return model
+
+
 def configure_logging(config):
     """Allow some basic logging configuration to be read from ini file.
 
@@ -1003,10 +1028,7 @@ class ConfiguresGalaxyMixin:
         """
         Preconditions: object_store must be set on self.
         """
-        if self.config.database_connection:
-            db_url = self.config.database_connection
-        else:
-            db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
+        db_url = get_database_url(self.config)
         install_db_url = self.config.install_database_connection
         # TODO: Consider more aggressive check here that this is not the same
         # database file under the hood.
@@ -1035,17 +1057,12 @@ class ConfiguresGalaxyMixin:
                 install_database_options = self.config.install_database_engine_options
             verify_tools(self, install_db_url, config_file, install_database_options)
 
-        from galaxy.model import mapping
-        self.model = mapping.init(self.config.file_path,
-                                  db_url,
-                                  self.config.database_engine_options,
-                                  map_install_models=combined_install_database,
-                                  database_query_profiling_proxy=self.config.database_query_profiling_proxy,
-                                  object_store=self.object_store,
-                                  trace_logger=getattr(self, "trace_logger", None),
-                                  use_pbkdf2=self.config.get_bool('use_pbkdf2', True),
-                                  slow_query_log_threshold=self.config.slow_query_log_threshold)
-
+        self.model = init_models_from_config(
+            self.config,
+            map_install_models=combined_install_database,
+            object_store=self.object_store,
+            trace_logger=getattr(self, "trace_logger", None)
+        )
         if combined_install_database:
             log.info("Install database targetting Galaxy's database configuration.")
             self.install_model = self.model

lib/galaxy/model/orm/scripts.py

@@ -1,11 +1,13 @@
 """
 Code to support database helper scripts (create_db.py, manage_db.py, etc...).
 """
+import argparse
 import logging
 import os.path
 
 from galaxy.util.path import get_ext
 from galaxy.util.properties import find_config_file, load_app_properties
+from galaxy.util.script import populate_config_args
 
 
 log = logging.getLogger(__name__)
@@ -45,17 +47,36 @@ DATABASE = {
 }
 
 
-def read_config_file_arg(argv, config_names, cwd=None):
-    if '-c' in argv:
-        pos = argv.index('-c')
-        argv.pop(pos)
-        return argv.pop(pos)
-    if cwd:
-        cwd = [cwd, os.path.join(cwd, 'config')]
-    return find_config_file(config_names, dirs=cwd)
+def _read_model_arguments(argv, use_argparse=False):
+    if use_argparse:
+        parser = argparse.ArgumentParser()
+        parser.add_argument('database', metavar='DATABASE', type=str,
+                            default="galaxy",
+                            nargs='?',
+                            help='database to target (galaxy, tool_shed, install)')
+        populate_config_args(parser)
+        args = parser.parse_args(argv[1:] if argv else [])
+        return args.config_file, args.config_section, args.database
+    else:
+        config_file = None
+        for arg in ["-c", "--config", "--config-file"]:
+            if arg in argv:
+                pos = argv.index(arg)
+                argv.pop(pos)
+                config_file = argv.pop(pos)
+        config_section = None
+        if "--config-section" in argv:
+            pos = argv.index("--config-section")
+            argv.pop(pos)
+            config_section = argv.pop(pos)
+        if argv and (argv[-1] in DATABASE):
+            database = argv.pop()  # database name tool_shed, galaxy, or install.
+        else:
+            database = 'galaxy'
+        return config_file, config_section, database
 
 
-def get_config(argv, cwd=None):
+def get_config(argv, use_argparse=True, cwd=None):
     """
     Read sys.argv and parse out repository of migrations and database url.
 
@@ -84,23 +105,24 @@ def get_config(argv, cwd=None):
     'lib/galaxy/model/migrate'
     >>> rmtree(config_dir)
     """
-    if argv and (argv[-1] in DATABASE):
-        database = argv.pop()  # database name tool_shed, galaxy, or install.
-    else:
-        database = 'galaxy'
+    config_file, config_section, database = _read_model_arguments(argv, use_argparse=use_argparse)
     database_defaults = DATABASE[database]
+    if config_file is None:
+        config_names = database_defaults.get('config_names', DEFAULT_CONFIG_NAMES)
+        if cwd:
+            cwd = [cwd, os.path.join(cwd, 'config')]
+        config_file = find_config_file(config_names, dirs=cwd)
 
-    config_names = database_defaults.get('config_names', DEFAULT_CONFIG_NAMES)
-    config_file = read_config_file_arg(argv, config_names, cwd=cwd)
     repo = database_defaults['repo']
     config_prefix = database_defaults.get('config_prefix', DEFAULT_CONFIG_PREFIX)
     config_override = database_defaults.get('config_override', 'GALAXY_CONFIG_')
     default_sqlite_file = database_defaults['default_sqlite_file']
-    if not config_file or get_ext(config_file, ignore='sample') == 'yaml':
-        config_section = database_defaults.get('config_section', None)
-    else:
-        # An .ini file - just let load_app_properties find app:main.
-        config_section = None
+    if config_section is None:
+        if not config_file or get_ext(config_file, ignore='sample') == 'yaml':
+            config_section = database_defaults.get('config_section', None)
+        else:
+            # Just use the default found by load_app_properties.
+            config_section = None
     properties = load_app_properties(config_file=config_file, config_prefix=config_override, config_section=config_section)
 
     if ("%sdatabase_connection" % config_prefix) in properties:

lib/galaxy/util/hash_util.py

@@ -10,11 +10,30 @@ import logging
 
 log = logging.getLogger(__name__)
 
+BLOCK_SIZE = 1024 * 1024
+
 sha1 = hashlib.sha1
+sha256 = hashlib.sha256
 sha = sha1
 md5 = hashlib.md5
 
 
+def memory_bound_hexdigest(hash_func, path=None, file=None):
+    hasher = hash_func()
+    if file is None:
+        assert path is not None
+        file = open(path, "rb")
+    else:
+        assert path is None, "Cannot specify path and path keyword arguments."
+
+    try:
+        for block in iter(lambda: file.read(BLOCK_SIZE), b''):
+                hasher.update(block)
+        return hasher.hexdigest()
+    finally:
+        file.close()
+
+
 def md5_hash_file(path):
     """
     Return a md5 hashdigest for a file or None if path could not be read.

lib/galaxy/util/script.py

@@ -11,6 +11,19 @@ ACTIONS = None
 ARGUMENTS = None
 DEFAULT_ACTION = None
 
+ARG_HELP_CONFIG_FILE = """
+Galaxy config file (defaults to $GALAXY_ROOT/config/galaxy.yml if that file exists
+or else to ./config/galaxy.ini if that exists). If this isn't set on the
+command line it can be set with the environment variable GALAXY_CONFIG_FILE.
+"""
+
+# ARG_HELP_CONFIG_SECTION = """
+# Section containing application configuration in the target config file specified with
+# -c/--config-file. This defaults to 'galaxy' for YAML/JSON configuration files and 'main'
+# with 'app:' prepended for INI. If this isn't set on the command line it can be set with
+# the environment variable GALAXY_CONFIG_SECTION.
+# """
+
 
 def main_factory(description=None, actions=None, arguments=None, default_action=None):
     global DESCRIPTION, ACTIONS, ARGUMENTS, DEFAULT_ACTION
@@ -26,19 +39,41 @@ def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]
     args = _arg_parser().parse_args(argv)
-    kwargs = _app_properties(args)
+    kwargs = app_properties_from_args(args)
     action = args.action
     action_func = ACTIONS[action]
     action_func(args, kwargs)
 
 
-def _app_properties(args):
-    # FIXME: you can use galaxy.util.path.extensions for this
-    config_file = args.config_file or find_config_file(args.app)
-    app_properties = load_app_properties(config_file=config_file, config_section=args.config_section)
+def app_properties_from_args(args, legacy_config_override=None, app=None):
+    config_file = config_file_from_args(args, legacy_config_override=legacy_config_override, app=app)
+    config_section = getattr(args, "config_section", None)
+    app_properties = load_app_properties(config_file=config_file, config_section=config_section)
     return app_properties
 
 
+def config_file_from_args(args, legacy_config_override=None, app=None):
+    app = app or getattr(args, "app", "galaxy")
+    config_file = legacy_config_override or args.config_file or find_config_file(app)
+    return config_file
+
+
+def populate_config_args(parser):
+    # config and config-file respected because we have used different arguments at different
+    # time for scripts.
+
+    # Options (e.g. option_name) not found in this file can have their defaults overridden
+    # set setting GALAXY_CONFIG_OPTION_NAME where OPTION_NAME is option_name converted to upper case.
+    # Options specified in that file can be overridden for this program set setting
+    # GALAXY_CONFIG_OVERRIDE_OPTION_NAME to a new value.
+    parser.add_argument("-c", "--config-file", "--config",
+                        default=os.environ.get('GALAXY_CONFIG_FILE', None),
+                        help=ARG_HELP_CONFIG_FILE)
+    parser.add_argument("--config-section",
+                        default=os.environ.get('GALAXY_CONFIG_SECTION', None),
+                        help=argparse.SUPPRESS)  # See ARG_HELP_CONFIG_SECTION comment above for unsuppressed details.
+
+
 def _arg_parser():
     parser = argparse.ArgumentParser(description=DESCRIPTION)
     parser.add_argument('action', metavar='ACTION', type=str,
@@ -46,10 +81,7 @@ def _arg_parser():
                         default=DEFAULT_ACTION,
                         nargs='?' if DEFAULT_ACTION is not None else None,
                         help='action to perform')
-    parser.add_argument("-c", "--config-file",
-                        default=os.environ.get('GALAXY_CONFIG_FILE', None))
-    parser.add_argument("--config-section",
-                        default=os.environ.get('GALAXY_CONFIG_SECTION', None))
+    populate_config_args(parser)
     parser.add_argument("--app",
                         default=os.environ.get('GALAXY_APP', 'galaxy'))
     for argument in ARGUMENTS:

scripts/cleanup_datasets/admin_cleanup_datasets.py

@@ -38,6 +38,7 @@ Author: Lance Parsons (lparsons@princeton.edu)
 """
 from __future__ import print_function
 
+import argparse
 import logging
 import os
 import shutil
@@ -45,12 +46,10 @@ import sys
 import time
 from collections import defaultdict
 from datetime import datetime, timedelta
-from optparse import OptionParser
 from time import strftime
 
 import sqlalchemy as sa
 from mako.template import Template
-from six.moves import configparser
 from sqlalchemy import and_, false
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
@@ -58,6 +57,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa
 import galaxy.config
 import galaxy.model.mapping
 import galaxy.util
+from galaxy.util.script import app_properties_from_args, populate_config_args
 
 from cleanup_datasets import CleanupDatasetsApplication  # noqa: I100
 
@@ -74,59 +74,59 @@ def main():
     contains the specified text will be marked as deleted in user's history and
     the user will be notified by email using the specified template file.
     """
-    usage = "usage: %prog [options] galaxy.ini"
-    parser = OptionParser(usage=usage)
-    parser.add_option("-d", "--days", dest="days", action="store",
-                      type="int", help="number of days (60)", default=60)
-    parser.add_option("--tool_id", default=None,
-                      help="Text to match against tool_id"
-                      "Default: match all")
-    parser.add_option("--template", default=None,
-                      help="Mako Template file to use as email "
-                      "Variables are 'cutoff' for the cutoff in days, "
-                      "'email' for users email and "
-                      "'datasets' which is a list of tuples "
-                      "containing 'dataset' and 'history' names. "
-                      "Default: admin_cleanup_deletion_template.txt")
-    parser.add_option("-i", "--info_only", action="store_true",
-                      dest="info_only", help="info about the requested action",
-                      default=False)
-    parser.add_option("-e", "--email_only", action="store_true",
-                      dest="email_only", help="Send emails only, don't delete",
-                      default=False)
-    parser.add_option("--smtp", default=None,
-                      help="SMTP Server to use to send email. "
-                      "Default: [read from galaxy ini file]")
-    parser.add_option("--fromaddr", default=None,
-                      help="From address to use to send email. "
-                      "Default: [read from galaxy ini file]")
-    (options, args) = parser.parse_args()
-    if len(args) != 1:
-        parser.print_help()
-        sys.exit()
-    ini_file = args[0]
+    parser = argparse.ArgumentParser()
+    parser.add_argument('legacy_config', metavar='CONFIG', type=str,
+                        default=None,
+                        nargs='?',
+                        help='config file (legacy, use --config instead)')
+    parser.add_argument("-d", "--days", dest="days", action="store",
+                        type=int, help="number of days (60)", default=60)
+    parser.add_argument("--tool_id", default=None,
+                        help="Text to match against tool_id"
+                        "Default: match all")
+    parser.add_argument("--template", default=None,
+                        help="Mako Template file to use as email "
+                        "Variables are 'cutoff' for the cutoff in days, "
+                        "'email' for users email and "
+                        "'datasets' which is a list of tuples "
+                        "containing 'dataset' and 'history' names. "
+                        "Default: admin_cleanup_deletion_template.txt")
+    parser.add_argument("-i", "--info_only", action="store_true",
+                        dest="info_only", help="info about the requested action",
+                        default=False)
+    parser.add_argument("-e", "--email_only", action="store_true",
+                        dest="email_only", help="Send emails only, don't delete",
+                        default=False)
+    parser.add_argument("--smtp", default=None,
+                        help="SMTP Server to use to send email. "
+                        "Default: [read from galaxy ini file]")
+    parser.add_argument("--fromaddr", default=None,
+                        help="From address to use to send email. "
+                        "Default: [read from galaxy ini file]")
+    populate_config_args(parser)
 
-    config_parser = configparser.ConfigParser({'here': os.getcwd()})
-    config_parser.read(ini_file)
-    config_dict = {}
-    for key, value in config_parser.items("app:main"):
-        config_dict[key] = value
+    args = parser.parse_args()
+    config_override = None
+    if args.legacy_config:
+        config_override = args.legacy_config
 
-    if options.smtp is not None:
-        config_dict['smtp_server'] = options.smtp
-    if config_dict.get('smtp_server') is None:
+    app_properties = app_properties_from_args(args, legacy_config_override=config_override)
+
+    if args.smtp is not None:
+        app_properties['smtp_server'] = args.smtp
+    if app_properties.get('smtp_server') is None:
         parser.error("SMTP Server must be specified as an option (--smtp) "
                      "or in the config file (smtp_server)")
 
-    if options.fromaddr is not None:
-        config_dict['email_from'] = options.fromaddr
-    if config_dict.get('email_from') is None:
+    if args.fromaddr is not None:
+        app_properties['email_from'] = args.fromaddr
+    if app_properties.get('email_from') is None:
         parser.error("From address must be specified as an option "
                      "(--fromaddr) or in the config file "
                      "(email_from)")
 
     scriptdir = os.path.dirname(os.path.abspath(__file__))
-    template_file = options.template
+    template_file = args.template
     if template_file is None:
         default_template = os.path.join(scriptdir,
                                         'admin_cleanup_deletion_template.txt')
@@ -145,24 +145,24 @@ def main():
     elif not os.path.exists(template_file):
         parser.error("Specified template file (%s) not found." % template_file)
 
-    config = galaxy.config.Configuration(**config_dict)
+    config = galaxy.config.Configuration(**app_properties)
 
     app = CleanupDatasetsApplication(config)
-    cutoff_time = datetime.utcnow() - timedelta(days=options.days)
+    cutoff_time = datetime.utcnow() - timedelta(days=args.days)
     now = strftime("%Y-%m-%d %H:%M:%S")
 
     print("##########################################")
-    print("\n# %s - Handling stuff older than %i days" % (now, options.days))
+    print("\n# %s - Handling stuff older than %i days" % (now, args.days))
 
-    if options.info_only:
+    if args.info_only:
         print("# Displaying info only ( --info_only )\n")
-    elif options.email_only:
+    elif args.email_only:
         print("# Sending emails only, not deleting ( --email_only )\n")
 
     administrative_delete_datasets(
-        app, cutoff_time, options.days, tool_id=options.tool_id,
+        app, cutoff_time, args.days, tool_id=args.tool_id,
         template_file=template_file, config=config,
-        email_only=options.email_only, info_only=options.info_only)
+        email_only=args.email_only, info_only=args.info_only)
     app.shutdown()
     sys.exit(0)
 

scripts/cleanup_datasets/cleanup_datasets.py

@@ -1,28 +1,27 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
+import argparse
 import logging
 import os
 import shutil
 import sys
 import time
 from datetime import datetime, timedelta
-from optparse import OptionParser
 from time import strftime
 
 import sqlalchemy as sa
-from six.moves import configparser
 from sqlalchemy import and_, false, null, true
 from sqlalchemy.orm import eagerload
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
 
 import galaxy.config
-import galaxy.model.mapping
 from galaxy.datatypes.registry import Registry
 from galaxy.exceptions import ObjectNotFound
 from galaxy.objectstore import build_object_store_from_config
 from galaxy.util import unicodify
+from galaxy.util.script import app_properties_from_args, populate_config_args
 
 log = logging.getLogger()
 log.setLevel(logging.INFO)
@@ -69,68 +68,65 @@ def main():
     Another way of stating it is: LibraryDatasetDatasetAssociation objects map LibraryDataset objects to Dataset objects,
     and Dataset objects may be mapped to History objects via HistoryDatasetAssociation objects.
     """
-    usage = "usage: %prog [options] galaxy.ini"
-    parser = OptionParser(usage=usage)
-    parser.add_option("-d", "--days", dest="days", action="store", type="int", help="number of days (60)", default=60)
-    parser.add_option("-r", "--remove_from_disk", action="store_true", dest="remove_from_disk", help="remove datasets from disk when purged", default=False)
-    parser.add_option("-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False)
-    parser.add_option("-f", "--force_retry", action="store_true", dest="force_retry", help="performs the requested actions, but ignores whether it might have been done before. Useful when -r wasn't used, but should have been", default=False)
-    parser.add_option("-1", "--delete_userless_histories", action="store_true", dest="delete_userless_histories", default=False, help="delete userless histories and datasets")
-    parser.add_option("-2", "--purge_histories", action="store_true", dest="purge_histories", default=False, help="purge deleted histories")
-    parser.add_option("-3", "--purge_datasets", action="store_true", dest="purge_datasets", default=False, help="purge deleted datasets")
-    parser.add_option("-4", "--purge_libraries", action="store_true", dest="purge_libraries", default=False, help="purge deleted libraries")
-    parser.add_option("-5", "--purge_folders", action="store_true", dest="purge_folders", default=False, help="purge deleted library folders")
-    parser.add_option("-6", "--delete_datasets", action="store_true", dest="delete_datasets", default=False, help="mark deletable datasets as deleted and purge associated dataset instances")
+    parser = argparse.ArgumentParser()
+    parser.add_argument('legacy_config', metavar='CONFIG', type=str,
+                        default=None,
+                        nargs='?',
+                        help='config file (legacy, use --config instead)')
+    parser.add_argument("-d", "--days", dest="days", action="store", type=int, help="number of days (60)", default=60)
+    parser.add_argument("-r", "--remove_from_disk", action="store_true", dest="remove_from_disk", help="remove datasets from disk when purged", default=False)
+    parser.add_argument("-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False)
+    parser.add_argument("-f", "--force_retry", action="store_true", dest="force_retry", help="performs the requested actions, but ignores whether it might have been done before. Useful when -r wasn't used, but should have been", default=False)
+    parser.add_argument("-1", "--delete_userless_histories", action="store_true", dest="delete_userless_histories", default=False, help="delete userless histories and datasets")
+    parser.add_argument("-2", "--purge_histories", action="store_true", dest="purge_histories", default=False, help="purge deleted histories")
+    parser.add_argument("-3", "--purge_datasets", action="store_true", dest="purge_datasets", default=False, help="purge deleted datasets")
+    parser.add_argument("-4", "--purge_libraries", action="store_true", dest="purge_libraries", default=False, help="purge deleted libraries")
+    parser.add_argument("-5", "--purge_folders", action="store_true", dest="purge_folders", default=False, help="purge deleted library folders")
+    parser.add_argument("-6", "--delete_datasets", action="store_true", dest="delete_datasets", default=False, help="mark deletable datasets as deleted and purge associated dataset instances")
+    populate_config_args(parser)
 
-    (options, args) = parser.parse_args()
-    if len(args) != 1:
-        parser.print_help()
-        sys.exit()
-    ini_file = args[0]
+    args = parser.parse_args()
+    config_override = None
+    if args.legacy_config:
+        config_override = args.legacy_config
 
-    if not (options.purge_folders ^ options.delete_userless_histories ^
-            options.purge_libraries ^ options.purge_histories ^
-            options.purge_datasets ^ options.delete_datasets):
+    if not (args.purge_folders ^ args.delete_userless_histories ^
+            args.purge_libraries ^ args.purge_histories ^
+            args.purge_datasets ^ args.delete_datasets):
         parser.print_help()
         sys.exit(0)
 
-    if options.remove_from_disk and options.info_only:
+    if args.remove_from_disk and args.info_only:
         parser.error("remove_from_disk and info_only are mutually exclusive")
 
-    config_parser = configparser.ConfigParser({'here': os.getcwd()})
-    config_parser.read(ini_file)
-    config_dict = {}
-    for key, value in config_parser.items("app:main"):
-        config_dict[key] = value
-
-    config = galaxy.config.Configuration(**config_dict)
-
+    app_properties = app_properties_from_args(args, legacy_config_override=config_override)
+    config = galaxy.config.Configuration(**app_properties)
     app = CleanupDatasetsApplication(config)
-    cutoff_time = datetime.utcnow() - timedelta(days=options.days)
+    cutoff_time = datetime.utcnow() - timedelta(days=args.days)
     now = strftime("%Y-%m-%d %H:%M:%S")
 
     print("##########################################")
-    print("\n# %s - Handling stuff older than %i days" % (now, options.days))
+    print("\n# %s - Handling stuff older than %i days" % (now, args.days))
 
-    if options.info_only:
+    if args.info_only:
         print("# Displaying info only ( --info_only )\n")
-    elif options.remove_from_disk:
+    elif args.remove_from_disk:
         print("Datasets will be removed from disk.\n")
     else:
         print("Datasets will NOT be removed from disk.\n")
 
-    if options.delete_userless_histories:
-        delete_userless_histories(app, cutoff_time, info_only=options.info_only, force_retry=options.force_retry)
-    elif options.purge_histories:
-        purge_histories(app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry)
-    elif options.purge_datasets:
-        purge_datasets(app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry)
-    elif options.purge_libraries:
-        purge_libraries(app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry)
-    elif options.purge_folders:
-        purge_folders(app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry)
-    elif options.delete_datasets:
-        delete_datasets(app, cutoff_time, options.remove_from_disk, info_only=options.info_only, force_retry=options.force_retry)
+    if args.delete_userless_histories:
+        delete_userless_histories(app, cutoff_time, info_only=args.info_only, force_retry=args.force_retry)
+    elif args.purge_histories:
+        purge_histories(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
+    elif args.purge_datasets:
+        purge_datasets(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
+    elif args.purge_libraries:
+        purge_libraries(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
+    elif args.purge_folders:
+        purge_folders(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
+    elif args.delete_datasets:
+        delete_datasets(app, cutoff_time, args.remove_from_disk, info_only=args.info_only, force_retry=args.force_retry)
 
     app.shutdown()
     sys.exit(0)
@@ -522,11 +518,9 @@ def _purge_folder(folder, app, remove_from_disk, info_only=False):
 class CleanupDatasetsApplication(object):
     """Encapsulates the state of a Universe application"""
     def __init__(self, config):
-        if config.database_connection is False:
-            config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database
         self.object_store = build_object_store_from_config(config)
         # Setup the database engine and ORM
-        self.model = galaxy.model.mapping.init(config.file_path, config.database_connection, engine_options={}, create_tables=False, object_store=self.object_store)
+        self.model = galaxy.config.init_models_from_config(config, object_store=self.object_store)
         registry = Registry()
         registry.load_datatypes()
         galaxy.model.set_datatypes_registry(registry)

scripts/cleanup_datasets/pgcleanup.py

@@ -6,16 +6,15 @@ pgcleanup.py - A script for cleaning up datasets in Galaxy efficiently, by
 """
 from __future__ import print_function
 
+import argparse
 import datetime
 import inspect
 import logging
 import os
 import shutil
 import sys
-from optparse import OptionParser
 
 import psycopg2
-from six.moves.configparser import ConfigParser
 from sqlalchemy.engine.url import make_url
 
 galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
@@ -25,6 +24,7 @@ import galaxy.config
 from galaxy.exceptions import ObjectNotFound
 from galaxy.objectstore import build_object_store_from_config
 from galaxy.util.bunch import Bunch
+from galaxy.util.script import app_properties_from_args, populate_config_args
 
 log = logging.getLogger()
 
@@ -39,7 +39,6 @@ class Dataset(Bunch):
 
 class Cleanup(object):
     def __init__(self):
-        self.options = None
         self.args = None
         self.config = None
         self.conn = None
@@ -61,49 +60,38 @@ class Cleanup(object):
                 self.action_names.append(name)
 
     def __parse_args(self):
-        default_config = os.path.abspath(os.path.join(galaxy_root, 'config', 'galaxy.ini'))
+        parser = argparse.ArgumentParser()
+        populate_config_args(parser)
+        parser.add_argument('-d', '--debug', action='store_true', dest='debug', help='Enable debug logging', default=False)
+        parser.add_argument('--dry-run', action='store_true', dest='dry_run', help="Dry run (rollback all transactions)", default=False)
+        parser.add_argument('--force-retry', action='store_true', dest='force_retry', help="Retry file removals (on applicable actions)", default=False)
+        parser.add_argument('-o', '--older-than', type=int, dest='days', help='Only perform action(s) on objects that have not been updated since the specified number of days', default=14)
+        parser.add_argument('-U', '--no-update-time', action='store_false', dest='update_time', help="Don't set update_time on updated objects", default=True)
+        parser.add_argument('-s', '--sequence', dest='sequence', help='Comma-separated sequence of actions, chosen from: %s' % self.action_names, default='')
+        parser.add_argument('-w', '--work-mem', dest='work_mem', help='Set PostgreSQL work_mem for this connection', default=None)
+        parser.add_argument('-l', '--log-dir', dest='log_dir', help='Log file directory', default=os.path.join(galaxy_root, 'scripts', 'cleanup_datasets'))
+        self.args = parser.parse_args()
 
-        parser = OptionParser()
-        parser.add_option('-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=default_config)
-        parser.add_option('-d', '--debug', action='store_true', dest='debug', help='Enable debug logging', default=False)
-        parser.add_option('--dry-run', action='store_true', dest='dry_run', help="Dry run (rollback all transactions)", default=False)
-        parser.add_option('--force-retry', action='store_true', dest='force_retry', help="Retry file removals (on applicable actions)", default=False)
-        parser.add_option('-o', '--older-than', type='int', dest='days', help='Only perform action(s) on objects that have not been updated since the specified number of days', default=14)
-        parser.add_option('-U', '--no-update-time', action='store_false', dest='update_time', help="Don't set update_time on updated objects", default=True)
-        parser.add_option('-s', '--sequence', dest='sequence', help='Comma-separated sequence of actions, chosen from: %s' % self.action_names, default='')
-        parser.add_option('-w', '--work-mem', dest='work_mem', help='Set PostgreSQL work_mem for this connection', default=None)
-        parser.add_option('-l', '--log-dir', dest='log_dir', help='Log file directory', default=os.path.join(galaxy_root, 'scripts', 'cleanup_datasets'))
-        (self.options, self.args) = parser.parse_args()
+        self.args.sequence = [x.strip() for x in self.args.sequence.split(',')]
 
-        self.options.sequence = [x.strip() for x in self.options.sequence.split(',')]
-
-        if self.options.sequence == ['']:
+        if self.args.sequence == ['']:
             print("Error: At least one action must be specified in the action sequence\n")
             parser.print_help()
             sys.exit(0)
 
     def __setup_logging(self):
         format = "%(funcName)s %(levelname)s %(asctime)s %(message)s"
-        if self.options.debug:
+        if self.args.debug:
             logging.basicConfig(level=logging.DEBUG, format=format)
         else:
             logging.basicConfig(level=logging.INFO, format=format)
 
     def __load_config(self):
-        log.info('Reading config from %s' % self.options.config)
-        config_parser = ConfigParser(dict(here=os.getcwd(),
-                                          database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE'))
-        config_parser.read(self.options.config)
-
-        config_dict = {}
-        for key, value in config_parser.items('app:main'):
-            config_dict[key] = value
-        config_dict['root_dir'] = galaxy_root
-
-        self.config = galaxy.config.Configuration(**config_dict)
+        app_properties = app_properties_from_args(self.args)
+        self.config = galaxy.config.Configuration(**app_properties)
 
     def __connect_db(self):
-        url = make_url(self.config.database_connection)
+        url = make_url(galaxy.config.get_database_url(self.config))
 
         log.info('Connecting to database with URL: %s' % url)
         args = url.translate_connect_args(username='user')
@@ -118,9 +106,9 @@ class Cleanup(object):
 
     def _open_logfile(self):
         action_name = inspect.stack()[1][3]
-        logname = os.path.join(self.options.log_dir, action_name + '.log')
+        logname = os.path.join(self.args.log_dir, action_name + '.log')
 
-        if self.options.dry_run:
+        if self.args.dry_run:
             log.debug('--dry-run specified, logging changes to stdout instead of log file: %s' % logname)
             self.logs[action_name] = sys.stdout
         else:
@@ -145,7 +133,7 @@ class Cleanup(object):
         self.logs[action_name].write(message.ljust(72, '='))
         self.logs[action_name].write('\n')
 
-        if self.options.dry_run:
+        if self.args.dry_run:
             log.debug('--dry-run specified, changes were logged to stdout insted of log file')
         else:
             log.debug('Closing log file: %s' % self.logs[action_name].name)
@@ -155,14 +143,14 @@ class Cleanup(object):
 
     def _run(self):
         ok = True
-        for name in self.options.sequence:
+        for name in self.args.sequence:
             if name not in self.action_names:
                 log.error('Unknown action in sequence: %s' % name)
                 ok = False
         if not ok:
             log.critical('Exiting due to previous error(s)')
             sys.exit(1)
-        for name in self.options.sequence:
+        for name in self.args.sequence:
             log.info('Calling %s' % name)
             self.__getattribute__(name)()
             log.info('Finished %s' % name)
@@ -188,7 +176,7 @@ class Cleanup(object):
 
         cur = self.conn.cursor()
 
-        if self.options.dry_run:
+        if self.args.dry_run:
             sql = "SELECT MAX(id) FROM cleanup_event;"
             cur.execute(sql)
             max_id = cur.fetchone()[0]
@@ -217,9 +205,9 @@ class Cleanup(object):
 
         cur = self.conn.cursor()
 
-        if self.options.work_mem is not None:
-            log.info('Setting work_mem to %s' % self.options.work_mem)
-            cur.execute('SET work_mem TO %s', (self.options.work_mem,))
+        if self.args.work_mem is not None:
+            log.info('Setting work_mem to %s' % self.args.work_mem)
+            cur.execute('SET work_mem TO %s', (self.args.work_mem,))
 
         log.info('Executing SQL')
         cur.execute(sql, args)
@@ -228,7 +216,7 @@ class Cleanup(object):
         return cur
 
     def _flush(self):
-        if self.options.dry_run:
+        if self.args.dry_run:
             self.conn.rollback()
             log.info("--dry-run specified, all changes rolled back")
         else:
@@ -245,7 +233,7 @@ class Cleanup(object):
             log.error('Unable to get MetadataFile %s filename: %s' % (id, e))
             return
 
-        if not self.options.dry_run:
+        if not self.args.dry_run:
             try:
                 os.unlink(filename)
             except Exception as e:
@@ -341,7 +329,7 @@ class Cleanup(object):
         """
         Mark deleted all "anonymous" Histories (not owned by a registered user) that are older than the specified number of days.
         """
-        log.info('Marking deleted all userless Histories older than %i days' % self.options.days)
+        log.info('Marking deleted all userless Histories older than %i days' % self.args.days)
 
         event_id = self._create_event()
 
@@ -364,12 +352,12 @@ class Cleanup(object):
         """
 
         update_time_sql = ''
-        if self.options.update_time:
+        if self.args.update_time:
             update_time_sql = """,
                                   update_time = NOW()"""
 
         sql = sql % (update_time_sql, '%s', '%s')
-        args = (self.options.days, event_id)
+        args = (self.args.days, event_id)
         cur = self._update(sql, args)
         self._flush()
 
@@ -385,7 +373,7 @@ class Cleanup(object):
         Mark deleted all ImplicitlyConvertedDatasetAssociations whose hda_parent_id is purged in this step.
         Mark purged all HistoryDatasetAssociations for which an ImplicitlyConvertedDatasetAssociation with matching hda_id is deleted in this step.
         """
-        log.info('Marking purged all deleted HistoryDatasetAssociations older than %i days' % self.options.days)
+        log.info('Marking purged all deleted HistoryDatasetAssociations older than %i days' % self.args.days)
 
         event_id = self._create_event()
 
@@ -457,16 +445,16 @@ class Cleanup(object):
                                 AND NOT purged"""
         update_time_sql = ""
 
-        if self.options.force_retry:
+        if self.args.force_retry:
             force_retry_sql = ""
         else:
             # only update time if not doing force retry (otherwise a lot of things would have their update times reset that were actually purged a long time ago)
-            if self.options.update_time:
+            if self.args.update_time:
                 update_time_sql = """,
                               update_time = NOW()"""
 
         sql = sql % (update_time_sql, force_retry_sql, '%s', update_time_sql, update_time_sql, update_time_sql, '%s', '%s', '%s', '%s')
-        args = (self.options.days, event_id, event_id, event_id, event_id)
+        args = (self.args.days, event_id, event_id, event_id, event_id)
         cur = self._update(sql, args)
         self._flush()
 
@@ -573,15 +561,15 @@ class Cleanup(object):
                                 AND NOT purged"""
         update_time_sql = ""
 
-        if self.options.force_retry:
+        if self.args.force_retry:
             force_retry_sql = ""
         else:
-            if self.options.update_time:
+            if self.args.update_time:
                 update_time_sql += """,
                                 update_time = NOW()"""
 
         sql = sql % (update_time_sql, force_retry_sql, '%s', update_time_sql, update_time_sql, update_time_sql, update_time_sql, '%s', '%s', '%s', '%s', '%s')
-        args = (self.options.days, event_id, event_id, event_id, event_id, event_id)
+        args = (self.args.days, event_id, event_id, event_id, event_id, event_id)
         cur = self._update(sql, args)
         self._flush()
 
@@ -627,12 +615,12 @@ class Cleanup(object):
         """
 
         update_time_sql = ""
-        if self.options.update_time:
+        if self.args.update_time:
             update_time_sql += """,
                                   update_time = NOW()"""
 
         sql = sql % (update_time_sql, '%s', '%s')
-        args = (self.options.days, event_id)
+        args = (self.args.days, event_id)
         cur = self._update(sql, args)
         self._flush()
 
@@ -676,12 +664,12 @@ class Cleanup(object):
         """
 
         update_time_sql = ""
-        if self.options.update_time:
+        if self.args.update_time:
             update_time_sql += """,
                                   update_time = NOW()"""
 
         sql = sql % (update_time_sql, '%s', '%s', '%s')
-        args = (self.options.days, self.options.days, event_id)
+        args = (self.args.days, self.args.days, event_id)
         cur = self._update(sql, args)
         self._flush()
 
@@ -721,15 +709,15 @@ class Cleanup(object):
                                   AND NOT purged"""
         update_time_sql = ""
 
-        if self.options.force_retry:
+        if self.args.force_retry:
             force_retry_sql = ""
         else:
-            if self.options.update_time:
+            if self.args.update_time:
                 update_time_sql = """,
                                   update_time = NOW()"""
 
         sql = sql % (update_time_sql, force_retry_sql, '%s', '%s')
-        args = (self.options.days, event_id)
+        args = (self.args.days, event_id)
         cur = self._update(sql, args)
         self._flush()
 
@@ -752,7 +740,7 @@ class Cleanup(object):
 
             # don't check for existence of the dataset, it should exist
             self._log('Removing from disk: %s' % filename)
-            if not self.options.dry_run:
+            if not self.args.dry_run:
                 try:
                     os.unlink(filename)
                 except Exception as e:
@@ -761,7 +749,7 @@ class Cleanup(object):
             # extra_files_dir is optional so it's checked first
             if extra_files_dir is not None and os.path.exists(extra_files_dir):
                 self._log('Removing from disk: %s' % extra_files_dir)
-                if not self.options.dry_run:
+                if not self.args.dry_run:
                     try:
                         shutil.rmtree(extra_files_dir)
                     except Exception as e:

scripts/cleanup_datasets/populate_uuid.py

@@ -8,33 +8,32 @@ script fixes datasets that were generated before the change.
 """
 from __future__ import print_function
 
+import argparse
+import os
 import sys
 import uuid
 
-from galaxy.model import mapping
-from galaxy.model.orm.scripts import get_config
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
 
-assert sys.version_info[:2] >= (2, 6)
+import galaxy.config
+from galaxy.util.script import app_properties_from_args, populate_config_args
 
-
-def usage(prog):
-    print("usage: %s galaxy.ini" % prog)
-    print("""
+DESCRIPTION = """
 Populates blank uuid fields in datasets with randomly generated values.
 
 Going forward, these ids will be generated for all new datasets. This
 script fixes datasets that were generated before the change.
-    """)
+"""
 
 
 def main():
-    if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help":
-        usage(sys.argv[0])
-        sys.exit()
-    ini_file = sys.argv.pop(1)
-    config = get_config(ini_file)
+    parser = argparse.ArgumentParser(DESCRIPTION)
+    populate_config_args(parser)
+    args = parser.parse_args()
 
-    model = mapping.init(ini_file, config['db_url'], create_tables=False)
+    app_properties = app_properties_from_args(args)
+    config = galaxy.config.Configuration(**app_properties)
+    model = galaxy.config.init_models_from_config(config)
 
     for row in model.context.query(model.Dataset):
         if row.uuid is None:
@@ -47,6 +46,7 @@ def main():
             row.uuid = uuid.uuid4()
             print("Setting Workflow:", row.id, " UUID to ", row.uuid)
     model.context.flush()
+    print("Complete")
 
 
 if __name__ == "__main__":

scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Removes a dataset file ( which was first renamed by appending _purged to the file name ) from disk.
-Usage: python remove_renamed_datasets_from_disk.py renamed.log
-"""
-from __future__ import print_function
-
-import os
-import sys
-
-assert sys.version_info[:2] >= (2, 6)
-
-
-def usage(prog):
-    print("usage: %s file" % prog)
-    print("""
-Removes a set of files from disk. The input file should contain a list of files
-to be deleted, one per line. The full path must be specified and must begin
-with /var/opt/galaxy.
-
-A log of files deleted is created in a file with the same name as that input but
-with .removed.log appended.
-    """)
-
-
-def main():
-    if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help":
-        usage(sys.argv[0])
-        sys.exit()
-    infile = sys.argv[1]
-    outfile = infile + ".removed.log"
-    out = open(outfile, 'w')
-
-    print("# The following renamed datasets have been removed from disk", file=out)
-    i = 0
-    removed_files = 0
-    for i, line in enumerate(open(infile)):
-        line = line.rstrip('\r\n')
-        if line and line.startswith('/var/opt/galaxy'):
-            try:
-                os.unlink(line)
-                print(line, file=out)
-                removed_files += 1
-            except Exception as exc:
-                print("# Error, exception " + str(exc) + " caught attempting to remove " + line, file=out)
-    print("# Removed " + str(removed_files) + " files", file=out)
-
-
-if __name__ == "__main__":
-    main()

scripts/cleanup_datasets/rename_purged_datasets.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python
-"""
-Renames a dataset file by appending _purged to the file name so that it can later be removed from disk.
-Usage: python rename_purged_datasets.py purge.log
-"""
-from __future__ import print_function
-
-import os
-import sys
-
-assert sys.version_info[:2] >= (2, 6)
-
-
-def usage(prog):
-    print("usage: %s file" % prog)
-    print("""
-Marks a set of files as purged and renames them. The input file should contain a
-list of files to be purged, one per line. The full path must be specified and
-must begin with /var/opt/galaxy.
-A log of files marked as purged is created in a file with the same name as that
-input but with _purged appended. The resulting files can finally be removed from
-disk with remove_renamed_datasets_from_disk.py, by supplying it with a list of
-them.
-    """)
-
-
-def main():
-    if len(sys.argv) != 2 or sys.argv == "-h" or sys.argv == "--help":
-        usage(sys.argv[0])
-        sys.exit()
-    infile = sys.argv[1]
-    outfile = infile + ".renamed.log"
-    out = open(outfile, 'w')
-
-    print("# The following renamed datasets can be removed from disk", file=out)
-    i = 0
-    renamed_files = 0
-    for i, line in enumerate(open(infile)):
-        line = line.rstrip('\r\n')
-        if line and line.startswith('/var/opt/galaxy'):
-            try:
-                purged_filename = line + "_purged"
-                os.rename(line, purged_filename)
-                print(purged_filename, file=out)
-                renamed_files += 1
-            except Exception as exc:
-                print("# Error, exception " + str(exc) + " caught attempting to rename " + purged_filename, file=out)
-    print("# Renamed " + str(renamed_files) + " files", file=out)
-
-
-if __name__ == "__main__":
-    main()

scripts/cleanup_datasets/update_dataset_size.py

@@ -2,6 +2,8 @@
 """
 Updates dataset.size column.
 Remember to backup your database before running.
+
+Deprecated - this doesn't work with modern Galaxy configurations options.
 """
 from __future__ import print_function
 

scripts/cleanup_datasets/update_metadata.py

@@ -4,6 +4,8 @@
 Updates metadata in the database to match rev 1891.
 
 Remember to backup your database before running.
+
+Deprecated - this doesn't work with modern Galaxy configurations options.
 """
 from __future__ import print_function
 

scripts/communication/communication_server.py

@@ -49,22 +49,26 @@ from flask_socketio import (
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
 
-from galaxy.model import mapping
-from galaxy.model.orm.scripts import get_config
-from galaxy.util.properties import load_app_properties
+import galaxy.config
 from galaxy.util.sanitize_html import sanitize_html
+from galaxy.util.script import app_properties_from_args, populate_config_args
 from galaxy.web.security import SecurityHelper
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
 
-# Get config file and load up SA session
-config = get_config(sys.argv)
-model = mapping.init('/tmp/', config['db_url'])
-sa_session = model.context.current
+parser = argparse.ArgumentParser(description='Real-time communication server for Galaxy.')
+parser.add_argument('--port', type=int, default="7070", help='Port number on which the server should run.')
+parser.add_argument('--host', default='localhost', help='Hostname of the communication server.')
+populate_config_args(parser)
+args = parser.parse_args()
 
 # With the config file we can load the full app properties
-app_properties = load_app_properties(ini_file=config['config_file'])
+app_properties = app_properties_from_args(args)
+
+config = galaxy.config.Configuration(**app_properties)
+model = galaxy.config.init_models_from_config(config)
+sa_session = model.context.current
 
 # We need the ID secret for configuring the security helper to decrypt
 # galaxysession cookies.
@@ -217,9 +221,4 @@ def leave(message):
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Real-time communication server for Galaxy.')
-    parser.add_argument('--port', type=int, default="7070", help='Port number on which the server should run.')
-    parser.add_argument('--host', default='localhost', help='Hostname of the communication server.')
-
-    args = parser.parse_args()
     socketio.run(app, host=args.host, port=args.port)

scripts/data_libraries/build_lucene_index.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-"""
-Build index for full-text lucene search of files in data libraries.
-
-Requires a full text search server and configuration settings in
-galaxy.ini. See the lucene settings in the data library search section for more
-details.
-
-Run from the ~/scripts/data_libraries directory:
-%sh build_lucene_index.sh
-"""
-import csv
-import os
-import sys
-
-import requests
-from six.moves.configparser import ConfigParser
-from six.moves.urllib.parse import urlencode
-
-sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
-
-import galaxy.model.mapping
-from galaxy import config, model
-
-
-def main(ini_file):
-    sa_session, gconfig = get_sa_session(ini_file)
-    max_size = float(gconfig.get("fulltext_max_size", 100)) * 1048576
-    ignore_exts = gconfig.get("fulltext_noindex_filetypes", "").split(",")
-    search_url = gconfig.get("fulltext_url", None)
-    if not search_url:
-        raise ValueError("Need to specify search functionality in galaxy.ini")
-    dataset_file = create_dataset_file(get_lddas(sa_session, max_size, ignore_exts))
-    try:
-        build_index(search_url, dataset_file)
-    finally:
-        if os.path.exists(dataset_file):
-            os.remove(dataset_file)
-
-
-def build_index(search_url, dataset_file):
-    url = "%s/index?%s" % (search_url, urlencode({"docfile": dataset_file}))
-    requests.put(url)
-
-
-def create_dataset_file(dataset_iter):
-    dataset_file = os.path.join(os.getcwd(), "full-text-search-files.csv")
-    out_handle = open(dataset_file, "w")
-    writer = csv.writer(out_handle)
-    for did, dfile, dname in dataset_iter:
-        writer.writerow([did, dfile, dname])
-    out_handle.close()
-    return dataset_file
-
-
-def get_lddas(sa_session, max_size, ignore_exts):
-    for ldda in sa_session.query(model.LibraryDatasetDatasetAssociation).filter_by(deleted=False):
-        if (float(ldda.dataset.get_size()) > max_size or ldda.extension in ignore_exts):
-            fname = ""
-        else:
-            fname = ldda.dataset.get_file_name()
-        yield ldda.id, fname, _get_dataset_metadata(ldda).replace("\n", " ")
-
-
-def _get_dataset_metadata(ldda):
-    """Retrieve descriptions and information associated with a dataset.
-    """
-    lds = ldda.library_dataset
-    folder_info = _get_folder_info(lds.folder)
-    lds_info = lds.get_info()
-    if lds_info and not lds_info.startswith("upload"):
-        lds_info = lds_info.replace("no info", "")
-    else:
-        lds_info = ""
-    return "%s %s %s %s %s" % (lds.name or "", lds_info, ldda.metadata.dbkey,
-                               ldda.message, folder_info)
-
-
-def _get_folder_info(folder):
-    """Get names and descriptions for all parent folders except top level.
-    """
-    folder_info = ""
-    if folder and folder.parent:
-        folder_info = _get_folder_info(folder.parent)
-        folder_info += " %s %s" % (
-            folder.name.replace("Unnamed folder", ""),
-            folder.description or "")
-    return folder_info
-
-
-def get_sa_session(ini_file):
-    conf_parser = ConfigParser({'here': os.getcwd()})
-    conf_parser.read(ini_file)
-    kwds = dict()
-    for key, value in conf_parser.items("app:main"):
-        kwds[key] = value
-    ini_config = config.Configuration(**kwds)
-    db_con = ini_config.database_connection
-    if not db_con:
-        db_con = "sqlite:///%s?isolation_level=IMMEDIATE" % ini_config.database
-    model = galaxy.model.mapping.init(ini_config.file_path,
-                                      db_con,
-                                      engine_options={},
-                                      create_tables=False)
-    return model.context.current, ini_config
-
-
-if __name__ == "__main__":
-    main(*sys.argv[1:])

scripts/data_libraries/build_lucene_index.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-cd `dirname $0`/../..
-python ./scripts/data_libraries/build_lucene_index.py ./config/galaxy.ini

scripts/data_libraries/build_whoosh_index.py

@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-"""
-Build index for full-text whoosh search of files in data libraries.
-
-Requires configuration settings in galaxy.ini. See the whoosh settings in the
-data library search section for more details.
-
-Run from the ~/scripts/data_libraries directory:
-%sh build_whoosh_index.sh
-"""
-from __future__ import print_function
-
-import os
-import sys
-
-from six import text_type
-from six.moves import configparser
-
-sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
-
-# Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether search is enabled.
-try:
-    from whoosh.filedb.filestore import FileStorage
-    from whoosh.fields import Schema, STORED, TEXT
-    whoosh_search_enabled = True
-    schema = Schema(id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT)
-    import galaxy.model.mapping
-    from galaxy import config, model
-except ImportError:
-    whoosh_search_enabled = False
-    schema = None
-
-
-def build_index(sa_session, whoosh_index_dir):
-    storage = FileStorage(whoosh_index_dir)
-    index = storage.create_index(schema)
-    writer = index.writer()
-
-    def to_unicode(a_basestr):
-        if not isinstance(a_basestr, text_type):
-            return text_type(a_basestr, 'utf-8')
-        else:
-            return a_basestr
-    lddas_indexed = 0
-    for id, name, info, dbkey, message in get_lddas(sa_session):
-        writer.add_document(id=id,
-                            name=to_unicode(name),
-                            info=to_unicode(info),
-                            dbkey=to_unicode(dbkey),
-                            message=to_unicode(message))
-        lddas_indexed += 1
-    writer.commit()
-    print("Number of active library datasets indexed: ", lddas_indexed)
-
-
-def get_lddas(sa_session):
-    for ldda in sa_session.query(model.LibraryDatasetDatasetAssociation).filter_by(deleted=False):
-        id = ldda.id
-        name = ldda.name
-        info = ldda.library_dataset.get_info()
-        if info and not info.startswith('upload'):
-            info = info.replace('no info', '')
-        else:
-            info = ''
-        dbkey = ldda.metadata.dbkey
-        if ldda.message:
-            message = ldda.message
-        else:
-            message = ''
-        yield id, name, info, dbkey, message
-
-
-def get_sa_session_and_needed_config_settings(ini_file):
-    conf_parser = configparser.ConfigParser({'here': os.getcwd()})
-    conf_parser.read(ini_file)
-    kwds = dict()
-    for key, value in conf_parser.items("app:main"):
-        kwds[key] = value
-    config_settings = config.Configuration(**kwds)
-    db_con = config_settings.database_connection
-    if not db_con:
-        db_con = "sqlite:///%s?isolation_level=IMMEDIATE" % config_settings.database
-    model = galaxy.model.mapping.init(config_settings.file_path, db_con, engine_options={}, create_tables=False)
-    return model.context.current, config_settings
-
-
-if __name__ == "__main__":
-    if whoosh_search_enabled:
-        ini_file = sys.argv[1]
-        sa_session, config_settings = get_sa_session_and_needed_config_settings(ini_file)
-        whoosh_index_dir = config_settings.whoosh_index_dir
-        build_index(sa_session, whoosh_index_dir)

scripts/data_libraries/build_whoosh_index.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-cd `dirname $0`/../..
-python ./scripts/data_libraries/build_whoosh_index.py ./config/galaxy.ini

scripts/db_shell.py

@@ -32,7 +32,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa
 
 from galaxy.datatypes.registry import Registry
 from galaxy.model import *  # noqa
-from galaxy.model import set_datatypes_registry  # More expclicit than `*` import
+from galaxy.model import set_datatypes_registry  # More explicit than `*` import
 from galaxy.model.mapping import init
 from galaxy.model.orm.scripts import get_config
 

scripts/galaxy-main

@@ -184,7 +184,7 @@ class GalaxyConfigBuilder(object):
     def populate_options(cls, arg_parser):
         arg_parser.add_argument("-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)")
         arg_parser.add_argument("--ini-path", default=None, help="DEPRECATED: use -c/--config-file")
-        arg_parser.add_argument("--app", default=None, help="app section in ini file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI")
+        arg_parser.add_argument("--app", default=None, help="app section in config file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI")
         arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonzie process", action="store_true")
         arg_parser.add_argument("--daemon-log-file", default=None, help="log file for daemon script ")
         arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)")

scripts/grt/export.py

@@ -7,7 +7,6 @@ import argparse
 import json
 import logging
 import os
-import subprocess
 import sys
 import tarfile
 import time
@@ -18,41 +17,33 @@ import yaml
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
 import galaxy
+import galaxy.app
 import galaxy.config
-from galaxy.model import mapping
 from galaxy.objectstore import build_object_store_from_config
-from galaxy.util.properties import load_app_properties
+from galaxy.util import hash_util
+from galaxy.util.script import app_properties_from_args, config_file_from_args, populate_config_args
 
 sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample'))
 default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml'))
 
 
-def _init(config, need_app=False):
-    if config.startswith('/'):
-        config_file = os.path.abspath(config)
-    else:
-        config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, config))
-
-    properties = load_app_properties(ini_file=config_file)
+def _init(args, need_app=False):
+    properties = app_properties_from_args(args)
     config = galaxy.config.Configuration(**properties)
     object_store = build_object_store_from_config(config)
     if not config.database_connection:
         logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.ini")
 
     if need_app:
+        config_file = config_file_from_args(args)
         app = galaxy.app.UniverseApplication(global_conf={'__file__': config_file, 'here': os.getcwd()})
     else:
         app = None
 
+    model = galaxy.config.init_models_from_config(config, object_store=object_store)
     return (
-        mapping.init(
-            config.file_path,
-            config.database_connection,
-            create_tables=False,
-            object_store=object_store
-        ),
+        model,
         object_store,
-        config.database_connection.split(':')[0],
         config,
         app
     )
@@ -167,7 +158,7 @@ def main(argv):
     parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('-r', '--report-directory', help='Directory to store reports in',
                         default=os.path.abspath(os.path.join('.', 'reports')))
-    parser.add_argument('-c', '--config', help='Path to GRT config file',
+    parser.add_argument('-g', '--grt-config', help='Path to GRT config file',
                         default=default_config)
     parser.add_argument("-l", "--loglevel", choices=['debug', 'info', 'warning', 'error', 'critical'],
                         help="Set the logging level", default='warning')
@@ -175,6 +166,7 @@ def main(argv):
                         help="Batch size for sql queries")
     parser.add_argument("-m", "--max-records", type=int, default=0,
                         help="Maximum number of records to include in a single report. This option should ONLY be used when reporting historical data. Setting this may require running GRT multiple times to capture all historical logs.")
+    populate_config_args(parser)
 
     args = parser.parse_args()
     logging.getLogger().setLevel(getattr(logging, args.loglevel.upper()))
@@ -189,7 +181,7 @@ def main(argv):
 
     annotate('init_start', 'Loading GRT configuration...')
     try:
-        with open(args.config) as handle:
+        with open(args.grt_config) as handle:
             config = yaml.safe_load(handle)
     except Exception:
         logging.info('Using default GRT configuration')
@@ -209,7 +201,7 @@ def main(argv):
         last_job_sent = -1
 
     annotate('galaxy_init', 'Loading Galaxy...')
-    model, object_store, engine, gxconfig, app = _init(config['galaxy_config'], need_app=config['grt']['share_toolbox'])
+    model, object_store, gxconfig, app = _init(args, need_app=config['grt']['share_toolbox'])
     # Galaxy overrides our logging level.
     logging.getLogger().setLevel(getattr(logging, args.loglevel.upper()))
     sa_session = model.context.current
@@ -345,10 +337,8 @@ def main(argv):
         os.unlink(REPORT_BASE + '.' + name + '.tsv')
 
     _times.append(('job_finish', time.time() - _start_time))
-    sha = subprocess.check_output(['sha256sum', REPORT_BASE + '.tar.gz'])
+    sha = hash_util.memory_bound_hexdigest(hash_util.sha256, REPORT_BASE + ".tar.gz")
     _times.append(('hash_finish', time.time() - _start_time))
-    # Strip out to space
-    sha = sha[0:sha.index(' ')]
 
     # Now serialize the individual report data.
     with open(REPORT_BASE + '.json', 'w') as handle:

scripts/grt/grt.yml.sample

@@ -1,7 +1,3 @@
-# Location of your galaxy config file. The radio telescope will need to
-# partially initialize a copy of galaxy.
-galaxy_config: config/galaxy.ini
-
 grt:
     # Go to https://telescope.galaxyproject.org to obtain an Instance ID and API key
     instance_id: 

scripts/grt/upload.py

@@ -20,7 +20,7 @@ def main(argv):
     parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('-r', '--report-directory', help='Directory in which reports are stored',
                         default=os.path.abspath(os.path.join('.', 'reports')))
-    parser.add_argument('-c', '--config', help='Path to GRT config file',
+    parser.add_argument('-g', '--grt-config', help='Path to GRT config file',
                         default=default_config)
     parser.add_argument("-l", "--loglevel", choices=['debug', 'info', 'warning', 'error', 'critical'],
                         help="Set the logging level", default='warning')
@@ -28,7 +28,7 @@ def main(argv):
 
     logging.info('Loading GRT configuration...')
     try:
-        with open(args.config) as handle:
+        with open(args.grt_config) as handle:
             config = yaml.safe_load(handle)
     except Exception:
         logging.info('Using default GRT configuration')

scripts/helper.py

@@ -6,61 +6,47 @@ returns the disk path of a dataset.
 """
 from __future__ import print_function
 
+import argparse
 import os
 import sys
-from optparse import OptionParser
-
-from six.moves.configparser import ConfigParser
 
 sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))
 
-from galaxy.model import mapping
+import galaxy.config
+from galaxy.util.script import app_properties_from_args, populate_config_args
 from galaxy.web import security
 
-default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'config/galaxy.ini'))
+parser = argparse.ArgumentParser()
+populate_config_args(parser)
+parser.add_argument('-e', '--encode-id', dest='encode_id', help='Encode an ID')
+parser.add_argument('-d', '--decode-id', dest='decode_id', help='Decode an ID')
+parser.add_argument('--hda', dest='hda_id', help='Display HistoryDatasetAssociation info')
+parser.add_argument('--ldda', dest='ldda_id', help='Display LibraryDatasetDatasetAssociation info')
+args = parser.parse_args()
 
-parser = OptionParser()
-parser.add_option('-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=default_config)
-parser.add_option('-e', '--encode-id', dest='encode_id', help='Encode an ID')
-parser.add_option('-d', '--decode-id', dest='decode_id', help='Decode an ID')
-parser.add_option('--hda', dest='hda_id', help='Display HistoryDatasetAssociation info')
-parser.add_option('--ldda', dest='ldda_id', help='Display LibraryDatasetDatasetAssociation info')
-(options, args) = parser.parse_args()
+app_properties = app_properties_from_args(args)
+config = galaxy.config.Configuration(**app_properties)
+helper = security.SecurityHelper(id_secret=app_properties.get('id_secret'))
+model = galaxy.config.init_models_from_config(config)
 
-try:
-    assert options.encode_id or options.decode_id or options.hda_id or options.ldda_id
-except Exception:
-    parser.print_help()
-    sys.exit(1)
+if args.encode_id:
+    print('Encoded "%s": %s' % (args.encode_id, helper.encode_id(args.encode_id)))
 
-options.config = os.path.abspath(options.config)
+if args.decode_id:
+    print('Decoded "%s": %s' % (args.decode_id, helper.decode_id(args.decode_id)))
 
-config = ConfigParser(dict(file_path='database/files',
-                           id_secret='USING THE DEFAULT IS NOT SECURE!',
-                           database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE'))
-config.read(options.config)
-
-helper = security.SecurityHelper(id_secret=config.get('app:main', 'id_secret'))
-model = mapping.init(config.get('app:main', 'file_path'), config.get('app:main', 'database_connection'), create_tables=False)
-
-if options.encode_id:
-    print('Encoded "%s": %s' % (options.encode_id, helper.encode_id(options.encode_id)))
-
-if options.decode_id:
-    print('Decoded "%s": %s' % (options.decode_id, helper.decode_id(options.decode_id)))
-
-if options.hda_id:
+if args.hda_id:
     try:
-        hda_id = int(options.hda_id)
+        hda_id = int(args.hda_id)
     except Exception:
-        hda_id = int(helper.decode_id(options.hda_id))
+        hda_id = int(helper.decode_id(args.hda_id))
     hda = model.context.current.query(model.HistoryDatasetAssociation).get(hda_id)
     print('HDA "%s" is Dataset "%s" at: %s' % (hda.id, hda.dataset.id, hda.file_name))
 
-if options.ldda_id:
+if args.ldda_id:
     try:
-        ldda_id = int(options.ldda_id)
+        ldda_id = int(args.ldda_id)
     except Exception:
-        ldda_id = int(helper.decode_id(options.ldda_id))
+        ldda_id = int(helper.decode_id(args.ldda_id))
     ldda = model.context.current.query(model.HistoryDatasetAssociation).get(ldda_id)
     print('LDDA "%s" is Dataset "%s" at: %s' % (ldda.id, ldda.dataset.id, ldda.file_name))

scripts/manage_db.py

@@ -12,7 +12,8 @@ from galaxy.model.orm.scripts import get_config
 
 
 def invoke_migrate_main():
-    config = get_config(sys.argv)
+    # Migrate has its own args, so cannot use argparse
+    config = get_config(sys.argv, use_argparse=False)
     db_url = config['db_url']
     repo = config['repo']
 

scripts/others/incorrect_gops_jobs.py

@@ -1,111 +0,0 @@
-#!/usr/bin/env python
-"""
-Fetch jobs using gops_intersect, gops_merge, gops_subtract, gops_complement, gops_coverage
-wherein the second dataset doesn't have chr, start and end in standard columns 1, 2 and 3.
-"""
-from __future__ import print_function
-
-import os
-import sys
-import tempfile
-
-import sqlalchemy as sa
-from six.moves.configparser import ConfigParser
-
-import galaxy.app
-import galaxy.model.mapping
-
-assert sys.version_info[:2] >= (2, 6)
-
-
-class TestApplication(object):
-    """Encapsulates the state of a Universe application"""
-    def __init__(self, database_connection=None, file_path=None):
-        print("python path is: " + ", ".join(sys.path), file=sys.stderr)
-        if database_connection is None:
-            raise Exception("CleanupDatasetsApplication requires a database_connection value")
-        if file_path is None:
-            raise Exception("CleanupDatasetsApplication requires a file_path value")
-        self.database_connection = database_connection
-        self.file_path = file_path
-        # Setup the database engine and ORM
-        self.model = galaxy.model.mapping.init(self.file_path, self.database_connection, engine_options={}, create_tables=False)
-
-
-def main():
-    ini_file = sys.argv[1]
-    conf_parser = ConfigParser({'here': os.getcwd()})
-    conf_parser.read(ini_file)
-    configuration = {}
-    for key, value in conf_parser.items("app:main"):
-        configuration[key] = value
-    database_connection = configuration['database_connection']
-    file_path = configuration['file_path']
-    app = TestApplication(database_connection=database_connection, file_path=file_path)
-    jobs = {}
-    try:
-        for job in app.model.Job.filter(sa.and_(app.model.Job.table.c.create_time.between('2008-05-23', '2008-11-29'),
-                                                app.model.Job.table.c.state == 'ok',
-                                                sa.or_(sa.and_(sa.or_(app.model.Job.table.c.tool_id == 'gops_intersect_1',
-                                                                      app.model.Job.table.c.tool_id == 'gops_subtract_1',
-                                                                      app.model.Job.table.c.tool_id == 'gops_coverage_1'),
-                                                               sa.not_(app.model.Job.table.c.command_line.like('%-2 1,2,3%'))),
-                                                       sa.and_(sa.or_(app.model.Job.table.c.tool_id == 'gops_complement_1',
-                                                                      app.model.Job.table.c.tool_id == 'gops_merge_1'),
-                                                               sa.not_(app.model.Job.table.c.command_line.like('%-1 1,2,3%')))))).all():
-            print("# processing job id %s" % str(job.id))
-            for jtoda in job.output_datasets:
-                print("# --> processing JobToOutputDatasetAssociation id %s" % str(jtoda.id))
-                hda = app.model.HistoryDatasetAssociation.get(jtoda.dataset_id)
-                print("# ----> processing HistoryDatasetAssociation id %s" % str(hda.id))
-                if not hda.deleted:
-                    # Probably don't need this check, since the job state should suffice, but...
-                    if hda.dataset.state == 'ok':
-                        history = app.model.History.get(hda.history_id)
-                        print("# ------> processing history id %s" % str(history.id))
-                        if history.user_id:
-                            cmd_line = str(job.command_line)
-                            new_output = tempfile.NamedTemporaryFile('w')
-                            if job.tool_id in ['gops_intersect_1', 'gops_subtract_1', 'gops_coverage_1']:
-                                new_cmd_line = " ".join(map(str, cmd_line.split()[:4])) + " " + new_output.name + " " + " ".join(map(str, cmd_line.split()[5:]))
-                                job_output = cmd_line.split()[4]
-                            else:
-                                new_cmd_line = " ".join(map(str, cmd_line.split()[:3])) + " " + new_output.name + " " + " ".join(map(str, cmd_line.split()[4:]))
-                                job_output = cmd_line.split()[3]
-                            try:
-                                os.system(new_cmd_line)
-                            except Exception:
-                                pass
-                            diff_status = os.system('diff %s %s >> /dev/null' % (new_output.name, job_output))
-                            if diff_status == 0:
-                                continue
-                            print("# --------> Outputs differ")
-                            user = app.model.User.get(history.user_id)
-                            jobs[job.id] = {}
-                            jobs[job.id]['hda_id'] = hda.id
-                            jobs[job.id]['hda_name'] = hda.name
-                            jobs[job.id]['hda_info'] = hda.info
-                            jobs[job.id]['history_id'] = history.id
-                            jobs[job.id]['history_name'] = history.name
-                            jobs[job.id]['history_update_time'] = history.update_time
-                            jobs[job.id]['user_email'] = user.email
-    except Exception as e:
-        print("# caught exception: %s" % e)
-
-    print("\n\n# Number of incorrect Jobs: %d\n\n" % (len(jobs)))
-    print("#job_id\thda_id\thda_name\thda_info\thistory_id\thistory_name\thistory_update_time\tuser_email")
-    for jid in jobs:
-        print('%s\t%s\t"%s"\t"%s"\t%s\t"%s"\t"%s"\t%s' %
-            (str(jid),
-             str(jobs[jid]['hda_id']),
-             jobs[jid]['hda_name'],
-             jobs[jid]['hda_info'],
-             str(jobs[jid]['history_id']),
-             jobs[jid]['history_name'],
-             jobs[jid]['history_update_time'],
-             jobs[jid]['user_email']))
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

scripts/others/incorrect_gops_jobs.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-cd `dirname $0`/../..
-python ./scripts/others/incorrect_gops_jobs.py ./config/galaxy.ini >> ./scripts/others/incorrect_gops_jobs.log

scripts/others/incorrect_gops_join_jobs.py

@@ -1,101 +0,0 @@
-#!/usr/bin/env python
-"""
-Fetch gops_join wherein the use specified minimum coverage is not 1.
-"""
-from __future__ import print_function
-
-import os
-import sys
-import tempfile
-
-import sqlalchemy as sa
-from six.moves.configparser import ConfigParser
-
-import galaxy.app
-import galaxy.model.mapping
-
-assert sys.version_info[:2] >= (2, 6)
-
-
-class TestApplication(object):
-    """Encapsulates the state of a Universe application"""
-    def __init__(self, database_connection=None, file_path=None):
-        print("python path is: " + ", ".join(sys.path), file=sys.stderr)
-        if database_connection is None:
-            raise Exception("CleanupDatasetsApplication requires a database_connection value")
-        if file_path is None:
-            raise Exception("CleanupDatasetsApplication requires a file_path value")
-        self.database_connection = database_connection
-        self.file_path = file_path
-        # Setup the database engine and ORM
-        self.model = galaxy.model.mapping.init(self.file_path, self.database_connection, engine_options={}, create_tables=False)
-
-
-def main():
-    ini_file = sys.argv[1]
-    conf_parser = ConfigParser({'here': os.getcwd()})
-    conf_parser.read(ini_file)
-    configuration = {}
-    for key, value in conf_parser.items("app:main"):
-        configuration[key] = value
-    database_connection = configuration['database_connection']
-    file_path = configuration['file_path']
-    app = TestApplication(database_connection=database_connection, file_path=file_path)
-    jobs = {}
-    try:
-        for job in app.model.Job.filter(sa.and_(app.model.Job.table.c.create_time < '2008-12-16',
-                                                app.model.Job.table.c.state == 'ok',
-                                                app.model.Job.table.c.tool_id == 'gops_join_1',
-                                                sa.not_(app.model.Job.table.c.command_line.like('%-m 1 %')))).all():
-            print("# processing job id %s" % str(job.id))
-            for jtoda in job.output_datasets:
-                print("# --> processing JobToOutputDatasetAssociation id %s" % str(jtoda.id))
-                hda = app.model.HistoryDatasetAssociation.get(jtoda.dataset_id)
-                print("# ----> processing HistoryDatasetAssociation id %s" % str(hda.id))
-                if not hda.deleted:
-                    # Probably don't need this check, since the job state should suffice, but...
-                    if hda.dataset.state == 'ok':
-                        history = app.model.History.get(hda.history_id)
-                        print("# ------> processing history id %s" % str(history.id))
-                        if history.user_id:
-                            cmd_line = str(job.command_line)
-                            new_output = tempfile.NamedTemporaryFile('w')
-                            new_cmd_line = " ".join(map(str, cmd_line.split()[:4])) + " " + new_output.name + " " + " ".join(map(str, cmd_line.split()[5:]))
-                            job_output = cmd_line.split()[4]
-                            try:
-                                os.system(new_cmd_line)
-                            except Exception:
-                                pass
-                            diff_status = os.system('diff %s %s >> /dev/null' % (new_output.name, job_output))
-                            if diff_status == 0:
-                                continue
-                            print("# --------> Outputs differ")
-                            user = app.model.User.get(history.user_id)
-                            jobs[job.id] = {}
-                            jobs[job.id]['hda_id'] = hda.id
-                            jobs[job.id]['hda_name'] = hda.name
-                            jobs[job.id]['hda_info'] = hda.info
-                            jobs[job.id]['history_id'] = history.id
-                            jobs[job.id]['history_name'] = history.name
-                            jobs[job.id]['history_update_time'] = history.update_time
-                            jobs[job.id]['user_email'] = user.email
-    except Exception as e:
-        print("# caught exception: %s" % e)
-
-    print("\n\n# Number of incorrect Jobs: %d\n\n" % (len(jobs)))
-    print("#job_id\thda_id\thda_name\thda_info\thistory_id\thistory_name\thistory_update_time\tuser_email")
-    for jid in jobs:
-        print('%s\t%s\t"%s"\t"%s"\t%s\t"%s"\t"%s"\t%s' %
-            (str(jid),
-             str(jobs[jid]['hda_id']),
-             jobs[jid]['hda_name'],
-             jobs[jid]['hda_info'],
-             str(jobs[jid]['history_id']),
-             jobs[jid]['history_name'],
-             jobs[jid]['history_update_time'],
-             jobs[jid]['user_email']))
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

scripts/others/incorrect_gops_join_jobs.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-cd `dirname $0`/../..
-python ./scripts/others/incorrect_gops_join_jobs.py ./config/galaxy.ini >> ./scripts/others/incorrect_gops_join_jobs.log

scripts/runtime_stats.py

@@ -32,14 +32,20 @@ Examples
 from __future__ import print_function
 
 import argparse
+import os
 import re
 import sys
 
 import numpy
 import psycopg2
-from six.moves import configparser
 from sqlalchemy.engine import url
 
+galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+sys.path.insert(1, os.path.join(galaxy_root, 'lib'))
+
+import galaxy.config
+from galaxy.util.script import app_properties_from_args, populate_config_args
+
 
 DATA_SOURCES = ('metrics', 'history')
 METRICS_SQL = """
@@ -78,7 +84,7 @@ def parse_arguments():
                         help='Use SQL `LIKE` operator to find '
                              'a shed-installed tool using the tool\'s '
                              '"short" id')
-    parser.add_argument('-c', '--config', help='Galaxy config file')
+    populate_config_args(parser)
     parser.add_argument('-d', '--debug',
                         action='store_true',
                         default=False,
@@ -110,14 +116,12 @@ def parse_arguments():
         print('ERROR: Data source `%s` unknown, valid source are: %s'
               % (args.source, ', '.join(DATA_SOURCES)))
 
-    if args.config:
-        cp = configparser.ConfigParser()
-        cp.readfp(open(args.config))
-        uri = cp.get('app:main', 'database_connection')
-        names = {'database': 'dbname', 'username': 'user'}
-        args.connect_args = url.make_url(uri).translate_connect_args(**names)
-    else:
-        args.connect_args = {}
+    app_properties = app_properties_from_args(args)
+    config = galaxy.config.Configuration(**app_properties)
+    uri = args.config.get_database_url(config)
+
+    names = {'database': 'dbname', 'username': 'user'}
+    args.connect_args = url.make_url(uri).translate_connect_args(**names)
 
     if args.debug:
         print('Got options:')

scripts/secret_decoder_ring.py

@@ -2,32 +2,36 @@
 """
 Script to encode/decode the IDs that galaxy exposes to users and admins.
 """
+import argparse
 import logging
 import os
 import sys
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
 
-from galaxy.model import mapping
-from galaxy.model.orm.scripts import get_config
-from galaxy.util.properties import load_app_properties
+from galaxy.util.script import app_properties_from_args, populate_config_args
 from galaxy.web.security import SecurityHelper
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
 
-# Get config file and load up SA session
-config = get_config(sys.argv)
-model = mapping.init('/tmp/', config['db_url'])
-sa_session = model.context.current
+parser = argparse.ArgumentParser()
+parser.add_argument('action', metavar='ACTION', type=str,
+                    default=None,
+                    help='decode|encode')
+parser.add_argument('value', metavar='VALUE', type=str,
+                    default=None,
+                    help='value to encode or decode')
+populate_config_args(parser)
+args = parser.parse_args()
 
-# With the config file we can load the full app properties
-app_properties = load_app_properties(ini_file=config['config_file'])
+app_properties = app_properties_from_args(args)
+helper = SecurityHelper(id_secret=app_properties.get('id_secret'))
 
 # We need the ID secret for configuring the security helper to decrypt
 # galaxysession cookies.
 if "id_secret" not in app_properties:
-    log.warning('No ID_SECRET specified. Please set the "id_secret" in your galaxy.ini.')
+    log.warning('No ID_SECRET specified. Please set the "id_secret" in your galaxy.yml.')
 
 id_secret = app_properties.get('id_secret', 'dangerous_default')
 
@@ -35,17 +39,10 @@ security_helper = SecurityHelper(id_secret=id_secret)
 # And get access to the models
 # Login manager to manage current_user functionality
 
-if len(sys.argv) != 3:
-    sys.stdout.write("python %s (encode|decode) value\n" % sys.argv[0])
-    sys.exit(1)
-
-action = sys.argv[1]
-value = sys.argv[2]
-
-if action == 'decode':
-    sys.stdout.write(security_helper.decode_guid(value.lstrip('F')))
-elif action == 'encode':
-    sys.stdout.write(security_helper.encode_guid(value))
+if args.action == 'decode':
+    sys.stdout.write(security_helper.decode_guid(args.value.lstrip('F')))
+elif args.action == 'encode':
+    sys.stdout.write(security_helper.encode_guid(args.value))
 else:
     sys.stdout.write("Unknown argument")
 sys.stdout.write('\n')

scripts/set_dataset_sizes.py

@@ -1,40 +1,28 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
+import argparse
 import os
 import sys
-from optparse import OptionParser
-
-from six.moves.configparser import ConfigParser
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
 
 import galaxy.config
-from galaxy.model import mapping
 from galaxy.objectstore import build_object_store_from_config
-default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'config/galaxy.ini'))
+from galaxy.util.script import app_properties_from_args, populate_config_args
 
-parser = OptionParser()
-parser.add_option('-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=default_config)
-(options, args) = parser.parse_args()
+parser = argparse.ArgumentParser()
+populate_config_args(parser)
+args = parser.parse_args()
 
 
 def init():
-    options.config = os.path.abspath(options.config)
-
-    config_parser = ConfigParser(dict(here=os.getcwd(),
-                                      database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE'))
-    config_parser.read(options.config)
-
-    config_dict = {}
-    for key, value in config_parser.items("app:main"):
-        config_dict[key] = value
-
-    config = galaxy.config.Configuration(**config_dict)
+    app_properties = app_properties_from_args(args)
+    config = galaxy.config.Configuration(**app_properties)
 
     object_store = build_object_store_from_config(config)
-    return (mapping.init(config.file_path, config.database_connection, create_tables=False, object_store=object_store),
-            object_store)
+    model = galaxy.config.init_models_from_config(config, object_store=object_store)
+    return model, object_store
 
 
 if __name__ == '__main__':

scripts/set_user_disk_usage.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
+import argparse
 import os
 import sys
-from optparse import OptionParser
-
-from six.moves.configparser import ConfigParser
 
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
 
@@ -13,41 +11,30 @@ import galaxy.config
 from galaxy.model.util import pgcalc
 from galaxy.objectstore import build_object_store_from_config
 from galaxy.util import nice_size
-
+from galaxy.util.script import app_properties_from_args, populate_config_args
 
 default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'config/galaxy.ini'))
 
-parser = OptionParser()
-parser.add_option('-c', '--config', dest='config', help='Path to Galaxy config file (config/galaxy.ini)', default=default_config)
-parser.add_option('-u', '--username', dest='username', help='Username of user to update', default='all')
-parser.add_option('-e', '--email', dest='email', help='Email address of user to update', default='all')
-parser.add_option('--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False)
-(options, args) = parser.parse_args()
+parser = argparse.ArgumentParser()
+parser.add_argument('-u', '--username', dest='username', help='Username of user to update', default='all')
+parser.add_argument('-e', '--email', dest='email', help='Email address of user to update', default='all')
+parser.add_argument('--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False)
+populate_config_args(parser)
+args = parser.parse_args()
 
 
 def init():
-    options.config = os.path.abspath(options.config)
-    if options.username == 'all':
-        options.username = None
-    if options.email == 'all':
-        options.email = None
 
-    config_parser = ConfigParser(dict(here=os.getcwd(),
-                                      database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE'))
-    config_parser.read(options.config)
+    if args.username == 'all':
+        args.username = None
+    if args.email == 'all':
+        args.email = None
 
-    config_dict = {}
-    for key, value in config_parser.items("app:main"):
-        config_dict[key] = value
-
-    config = galaxy.config.Configuration(**config_dict)
+    app_properties = app_properties_from_args(args)
+    config = galaxy.config.Configuration(**app_properties)
     object_store = build_object_store_from_config(config)
-
-    from galaxy.model import mapping
-
-    return (mapping.init(config.file_path, config.database_connection, create_tables=False, object_store=object_store),
-            object_store,
-            config.database_connection.split(':')[0])
+    engine = galaxy.config.get_database_url(config).split(":")[0]
+    return galaxy.config.init_models_from_config(config, object_store=object_store), object_store, engine
 
 
 def quotacheck(sa_session, users, engine):
@@ -62,7 +49,7 @@ def quotacheck(sa_session, users, engine):
             print('usage changed while calculating, trying again...')
             return quotacheck(sa_session, user, engine)
     else:
-        new = pgcalc(sa_session, user.id, dryrun=options.dryrun)
+        new = pgcalc(sa_session, user.id, dryrun=args.dryrun)
     # yes, still a small race condition between here and the flush
     print('old usage:', nice_size(current), 'change:', end=' ')
     if new in (current, None):
@@ -72,7 +59,7 @@ def quotacheck(sa_session, users, engine):
             print('+%s' % (nice_size(new - current)))
         else:
             print('-%s' % (nice_size(current - new)))
-        if not options.dryrun and engine not in ('postgres', 'postgresql'):
+        if not args.dryrun and engine not in ('postgres', 'postgresql'):
             user.set_disk_usage(new)
             sa_session.add(user)
             sa_session.flush()
@@ -83,7 +70,7 @@ if __name__ == '__main__':
     model, object_store, engine = init()
     sa_session = model.context.current
 
-    if not options.username and not options.email:
+    if not args.username and not args.email:
         user_count = sa_session.query(model.User).count()
         print('Processing %i users...' % user_count)
         for i, user in enumerate(sa_session.query(model.User).enable_eagerloads(False).yield_per(1000)):
@@ -92,10 +79,10 @@ if __name__ == '__main__':
         print('100% complete')
         object_store.shutdown()
         sys.exit(0)
-    elif options.username:
-        user = sa_session.query(model.User).enable_eagerloads(False).filter_by(username=options.username).first()
-    elif options.email:
-        user = sa_session.query(model.User).enable_eagerloads(False).filter_by(email=options.email).first()
+    elif args.username:
+        user = sa_session.query(model.User).enable_eagerloads(False).filter_by(username=args.username).first()
+    elif args.email:
+        user = sa_session.query(model.User).enable_eagerloads(False).filter_by(email=args.email).first()
     if not user:
         print('User not found')
         sys.exit(1)

scripts/sync_reports_config.py

@@ -1,3 +1,5 @@
+"""Deprecated - only works with older ini files.
+"""
 from sys import argv
 
 from six.moves.configparser import ConfigParser

scripts/transfer.py

@@ -3,6 +3,8 @@
 Downloads files to temp locations.  This script is invoked by the Transfer
 Manager (galaxy.jobs.transfer_manager) and should not normally be invoked by
 hand.
+
+This is deprecated - it only works with older ini configurations of Galaxy.
 """
 import json
 import logging

test/integration/test_scripts.py

@@ -0,0 +1,188 @@
+"""Integration tests for various scripts in scripts/.
+"""
+
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+
+import yaml
+from base import integration_util
+from base.populators import DatasetPopulator
+
+from galaxy.util import galaxy_directory
+
+
+def skip_unless_module(module):
+    available = True
+    try:
+        __import__(module)
+    except ImportError:
+        available = False
+    if available:
+        return lambda func: func
+    template = "Module %s could not be loaded, dependent test skipped."
+    return unittest.skip(template % module)
+
+
+class ScriptsIntegrationTestCase(integration_util.IntegrationTestCase):
+
+    def setUp(self):
+        super(ScriptsIntegrationTestCase, self).setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+        self.config_dir = tempfile.mkdtemp()
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls._raw_config = config
+
+    def test_helper(self):
+        script = "helper.py"
+        self._scripts_check_argparse_help(script)
+
+        history_id = self.dataset_populator.new_history()
+        dataset = self.dataset_populator.new_dataset(history_id, wait=True)
+        dataset_id = dataset["id"]
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file, "--decode-id", dataset_id])
+        assert "Decoded " in output
+
+    def test_cleanup(self):
+        script = "cleanup_datasets/cleanup_datasets.py"
+        self._scripts_check_argparse_help(script)
+
+        history_id = self.dataset_populator.new_history()
+        delete_response = self.dataset_populator._delete("histories/%s" % history_id)
+        assert delete_response.status_code == 200
+        assert delete_response.json()["purged"] is False
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file, "--days", "0", "--purge_histories"])
+        print(output)
+        history_response = self.dataset_populator._get("histories/%s" % history_id)
+        assert history_response.status_code == 200
+        assert history_response.json()["purged"] is True, history_response.json()
+
+    def test_pgcleanup(self):
+        self._skip_if_not_postgres()
+
+        script = "cleanup_datasets/pgcleanup.py"
+        self._scripts_check_argparse_help(script)
+
+        history_id = self.dataset_populator.new_history()
+        delete_response = self.dataset_populator._delete("histories/%s" % history_id)
+        assert delete_response.status_code == 200
+        assert delete_response.json()["purged"] is False
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file, "--older-than", "0", "--sequence", "purge_deleted_histories"])
+        print(output)
+        history_response = self.dataset_populator._get("histories/%s" % history_id)
+        assert history_response.status_code == 200
+        assert history_response.json()["purged"] is True, history_response.json()
+
+    def test_set_user_disk_usage(self):
+        script = "set_user_disk_usage.py"
+        self._scripts_check_argparse_help(script)
+
+        history_id = self.dataset_populator.new_history()
+        self.dataset_populator.new_dataset(history_id, wait=True)
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file])
+        # verify the script runs to completion without crashing
+        assert "100% complete" in output, output
+
+    def test_set_dataset_sizes(self):
+        script = "set_dataset_sizes.py"
+        self._scripts_check_argparse_help(script)
+
+        # TODO: change the size of the dataset and verify this works.
+        history_id = self.dataset_populator.new_history()
+        self.dataset_populator.new_dataset(history_id, wait=True)
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file])
+        # verify the script runs to completion without crashing
+        assert "Completed 100%" in output, output
+
+    def test_populate_uuid(self):
+        script = "cleanup_datasets/populate_uuid.py"
+        self._scripts_check_argparse_help(script)
+
+        history_id = self.dataset_populator.new_history()
+        self.dataset_populator.new_dataset(history_id, wait=True)
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file])
+        assert "Complete" in output
+
+    def test_grt_export(self):
+        script = "grt/export.py"
+        self._scripts_check_argparse_help(script)
+
+        history_id = self.dataset_populator.new_history()
+        self.dataset_populator.new_dataset(history_id, wait=True)
+        config_file = self.write_config_file()
+        grt_config_file = os.path.join(self.config_dir, "grt.yml")
+        with open(grt_config_file, "w") as f:
+            yaml.dump({"grt": {"share_toolbox": True}, "sanitization": {"tools": []}, "tool_params": {}}, f)
+        self._scripts_check_output(script, ["-c", config_file, "-g", grt_config_file, "-r", self.config_dir])
+        report_files = os.listdir(self.config_dir)
+        json_files = [j for j in report_files if j.endswith(".json")]
+        assert len(json_files) == 1, "Expected one json report file in [%s]" % json_files
+        json_file = os.path.join(self.config_dir, json_files[0])
+        with open(json_file, "r") as f:
+            export = json.load(f)
+        assert export["version"] == 1
+
+    def test_admin_cleanup_datasets(self):
+        self._scripts_check_argparse_help("cleanup_datasets/admin_cleanup_datasets.py")
+
+    @skip_unless_module("flask_socketio")
+    def test_communication_server(self):
+        self._scripts_check_argparse_help("communication/communication_server.py")
+
+    def test_secret_decoder_ring(self):
+        script = "secret_decoder_ring.py"
+        self._scripts_check_argparse_help(script)
+
+        config_file = self.write_config_file()
+        output = self._scripts_check_output(script, ["-c", config_file, "encode", "1"])
+        encoded_id = output.strip()
+
+        output = self._scripts_check_output(script, ["-c", config_file, "decode", encoded_id])
+        assert output.strip() == "1"
+
+    def test_database_scripts(self):
+        self._scripts_check_argparse_help("create_db.py")
+        self._scripts_check_argparse_help("manage_db.py")
+        # TODO: test creating a smaller database - e.g. tool install database based on fresh
+        # config file.
+
+    def test_runtime_stats(self):
+        self._skip_if_not_postgres()
+        self._scripts_check_argparse_help("runtime_stats.py")
+
+    def _skip_if_not_postgres(self):
+        if not self._app.config.database_connection.startswith("post"):
+            raise unittest.SkipTest("Test only valid for postgres")
+
+    def _scripts_check_argparse_help(self, script):
+        # Test imports and argparse repsonse to --help with 0 exit code.
+        output = self._scripts_check_output(script, ["--help"])
+        # Test -h, --help in printed output message.
+        assert "-h, --help" in output
+
+    def _scripts_check_output(self, script, args):
+        cwd = galaxy_directory()
+        cmd = ["python", os.path.join(cwd, "scripts", script)] + args
+        clean_env = {
+            "PATH": os.environ.get("PATH", None),
+        }  # Don't let testing environment variables interfere with config.
+        return subprocess.check_output(cmd, cwd=cwd, env=clean_env)
+
+    def write_config_file(self):
+        config_dir = self.config_dir
+        path = os.path.join(config_dir, "galaxy.yml")
+        self._test_driver.temp_directories.extend([config_dir])
+        with open(path, "w") as f:
+            yaml.dump({"galaxy": self._raw_config}, f)
+
+        return path