diff --git a/scripts/dump_library_paths.py b/scripts/dump_library_paths.py new file mode 100644 index 00000000000..4c9360650a6 --- /dev/null +++ b/scripts/dump_library_paths.py @@ -0,0 +1,158 @@ +"""Dump paths to library dataset files (e.g. for backups). Caveats: + + - Does not perform any permissions checks on library *contents* (but you can restrict to "public" libraries). + - Only the latest version of undeleted library datasets are dumped. + +To use with rsync: +$ python dump_library_paths.py [options] -o libfiles.txt --relative /srv/galaxy/datasets +$ rsync -arvPR --files-from=libfiles.txt /srv/galaxy/datasets backup@backuphost:/backup/galaxy/datasets + +Or all in one: +$ python dump_library_paths.py [options] --relative /srv/galaxy/datasets | rsync -arvPR \ + --files-from=- /srv/galaxy/datasets backuphost:/backup/galaxy/datasets +""" +import logging +import os +import sys + +from sqlalchemy import false, not_ + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) + +import galaxy.config +import galaxy.model.mapping +from galaxy.objectstore import build_object_store_from_config +from galaxy.util.script import main_factory + +DESCRIPTION = "Locate all datasets in libraries." +ARGUMENTS = ( + ( + ('-v', '--verbose'), + dict( + action='store_true', + default=False, + help='Verbose logging output', + ), + ), + ( + ('-o', '--output'), + dict( + default='stdout', + help='Write output to file', + ), + ), + ( + ('-p', '--public'), + dict( + action='store_true', + default=False, + help='Only dump files in "public" libraries' + ), + ), + ( + ('--relative',), + dict( + default=None, + help='Write paths relative to the given directory', + ), + ), + ( + ('--exists',), + dict( + action='store_true', + default=False, + help='Check for dataset existence, warn if it does not exist', + ), + ), +) + +logging.basicConfig(stream=sys.stderr) +log = logging.getLogger(__name__) + + +def _config_logging(args): + if args.verbose: + log.setLevel(logging.DEBUG) + + +def _get_libraries(args, model): + log.debug('Setting up query') + library_access_action = model.security_agent.permitted_actions.LIBRARY_ACCESS.action + query = model.context.query(model.Library) + query = query.filter(model.Library.table.c.deleted == false()) + if args.public: + restricted_library_ids = {lp.library_id for lp in ( + model.context.query(model.LibraryPermissions).filter( + model.LibraryPermissions.table.c.action == library_access_action + ).distinct())} + if restricted_library_ids: + query = query.filter(not_(model.Library.table.c.id.in_(restricted_library_ids))) + query = query.order_by(model.Library.table.c.name) + return query + + +def _walk_library(folder): + datasets = set() + for f in folder.folders: + datasets.update(_walk_library(f)) + for ld in folder.active_datasets: + datasets.add(ld.library_dataset_dataset_association.dataset) + return datasets + + +def _walk_libraries(args, model): + for library in _get_libraries(args, model): + for dataset in _walk_library(library.root_folder): + yield (library, dataset) + + +def _open_output(args): + if args.output == 'stdout': + return sys.stdout + else: + return open(args.output, 'w') + + +def _path(path, args): + if args.relative is not None: + return os.path.relpath(path, args.relative) + else: + return path + + +def _get_library_dataset_paths(args, kwargs): + _config_logging(args) + config = galaxy.config.Configuration(**kwargs) + object_store = build_object_store_from_config(config) + model = galaxy.model.mapping.init('/tmp/', kwargs.get('database_connection'), object_store=object_store) + output = _open_output(args) + last_library = None + log.debug('Beginning library walk') + for library, dataset in _walk_libraries(args, model): + if library != last_library: + log.info('Library: %s', library.name) + filename = object_store.get_filename(dataset) + files_dir = dataset.get_extra_files_path() + if (args.exists and object_store.exists(dataset)) or not args.exists: + output.write('%s\n' % _path(filename, args)) + elif args.exists: + log.warning('Missing %s', filename) + if files_dir and os.path.exists(files_dir): + output.write('%s\n' % _path(files_dir, args)) + last_library = library + output.close() + + +ACTIONS = { + "get_library_dataset_paths": _get_library_dataset_paths, +} + + +if __name__ == '__main__': + main = main_factory( + description=DESCRIPTION, + actions=ACTIONS, + arguments=ARGUMENTS, + default_action="get_library_dataset_paths" + ) + main()