From ca123a4f2674351e01235bab545559b299b99ff9 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:18 -0500 Subject: [PATCH 01/35] Add a safe_relpath util function for ensuring a path does not reference an absolute or parent directory --- lib/galaxy/util/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index bcf9579e76e..591421bcc7f 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -27,7 +27,7 @@ from galaxy.util import json from email.MIMEText import MIMEText -from os.path import relpath +from os.path import relpath, normpath from hashlib import md5 from itertools import izip @@ -1143,6 +1143,23 @@ galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") def galaxy_directory(): return os.path.abspath(galaxy_root_path) + +def safe_relpath(path): + """ + Given what we expect to be a relative path, determine whether the path + would exist inside the current directory. + + :type path: string + :param path: a path to check + :rtype: bool + :returns: ``True`` if path is relative and does not reference a path + in a parent directory, ``False`` otherwise. + """ + if path.startswith(os.sep) or normpath(path).startswith(os.pardir): + return False + return True + + if __name__ == '__main__': import doctest doctest.testmod(sys.modules[__name__], verbose=False) From 3b963226a110b8de8aa11d3c4e053a220bbcba53 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:18 -0500 Subject: [PATCH 02/35] Security fixes for history imports --- lib/galaxy/exceptions/__init__.py | 5 ++ lib/galaxy/exceptions/error_codes.json | 7 ++- lib/galaxy/tools/imp_exp/__init__.py | 6 ++- .../tools/imp_exp/unpack_tar_gz_archive.py | 53 +++++++++++++------ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index e803bcb98e6..cda45693a10 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -66,6 +66,11 @@ class MalformedId( MessageException ): err_code = error_codes.MALFORMED_ID +class MalformedContents( MessageException ): + status_code = 400 + err_code = error_codes.MALFORMED_CONTENTS + + class UnknownContentsType( MessageException ): status_code = 400 err_code = error_codes.UNKNOWN_CONTENTS_TYPE diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 2ff18ad50d1..3b203fb1c85 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -56,9 +56,14 @@ }, { "name": "USER_TOOL_META_PARAMETER_PROBLEM", - "code": 400011, + "code": 400012, "message": "Supplied incorrect or incompatible tool meta parameters." }, + { + "name": "MALFORMED_CONTENTS", + "code": 400013, + "message": "The contents of the request are malformed." + }, { "name": "USER_AUTHENTICATION_FAILED", "code": 401001, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 1983bb2f58b..9ab14140640 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,6 +5,7 @@ import tempfile import json import datetime from galaxy import model +from galaxy.exceptions import MalformedContents from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode from galaxy.model.item_attrs import UsesAnnotations @@ -187,9 +188,9 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if dataset_attrs.get('exported', True) == True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ - os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) + os.path.realpath( os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) ) if not file_in_dir( temp_dataset_file_name, os.path.join( archive_dir, "datasets" ) ): - raise Exception( "Invalid dataset path: %s" % temp_dataset_file_name ) + raise MalformedContents( "Invalid dataset path: %s" % temp_dataset_file_name ) if datasets_usage_counts[ temp_dataset_file_name ] == 1: shutil.move( temp_dataset_file_name, hda.file_name ) else: @@ -313,6 +314,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + raise class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index d6890257c8c..f3dab0c5ae4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,6 +6,7 @@ usage: %prog archive_source dest_dir --[url|file] source type, either a URL or a file. """ +import os import sys import optparse import tarfile @@ -43,6 +44,22 @@ def url_to_file( url, dest_file ): return None +def check_archive( archive_file, dest_dir ): + """ + Ensure that a tar archive has no absolute paths or relative paths outside + the archive. + """ + with tarfile.open( archive_file, mode='r:gz' ) as archive_fp: + for arc_path in archive_fp.getnames(): + assert os.path.normpath( + os.path.join( + dest_dir, + arc_path + ) ).startswith( dest_dir.rstrip(os.sep) + os.sep ), \ + "Archive member would extract outside target directory: %s" % arc_path + return True + + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -51,13 +68,8 @@ def unpack_archive( archive_file, dest_dir ): archive_fp.extractall( path=dest_dir ) archive_fp.close() -if __name__ == "__main__": - # Parse command line. - parser = optparse.OptionParser() - parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) - parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) - (options, args) = parser.parse_args() + +def main(options, args): is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args @@ -66,14 +78,25 @@ if __name__ == "__main__": archive_source = b64decode( archive_source ) dest_dir = b64decode( dest_dir ) - try: - # Get archive from URL. - if is_url: - archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) - elif is_file: - archive_file = archive_source + # Get archive from URL. + if is_url: + archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) + elif is_file: + archive_file = archive_source - # Unpack archive. - unpack_archive( archive_file, dest_dir ) + # Unpack archive. + check_archive( archive_file, dest_dir ) + unpack_archive( archive_file, dest_dir ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) + parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) + parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) + (options, args) = parser.parse_args() + try: + main(options, args) except Exception, e: print "Error unpacking tar/gz archive: %s" % e, sys.stderr From 5ab53c3fda3a43236b444a31aa408c9f5cb5c4c4 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 03/35] Add a safe_relpath util function for ensuring a path does not reference an absolute or parent directory --- lib/galaxy/util/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index bcf9579e76e..591421bcc7f 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -27,7 +27,7 @@ from galaxy.util import json from email.MIMEText import MIMEText -from os.path import relpath +from os.path import relpath, normpath from hashlib import md5 from itertools import izip @@ -1143,6 +1143,23 @@ galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") def galaxy_directory(): return os.path.abspath(galaxy_root_path) + +def safe_relpath(path): + """ + Given what we expect to be a relative path, determine whether the path + would exist inside the current directory. + + :type path: string + :param path: a path to check + :rtype: bool + :returns: ``True`` if path is relative and does not reference a path + in a parent directory, ``False`` otherwise. + """ + if path.startswith(os.sep) or normpath(path).startswith(os.pardir): + return False + return True + + if __name__ == '__main__': import doctest doctest.testmod(sys.modules[__name__], verbose=False) From 88277c9723c558e2b2003f1b81c7f2d16143fcc6 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 04/35] Security fixes for history imports --- lib/galaxy/exceptions/__init__.py | 5 ++ lib/galaxy/exceptions/error_codes.json | 7 ++- lib/galaxy/tools/imp_exp/__init__.py | 6 ++- .../tools/imp_exp/unpack_tar_gz_archive.py | 53 +++++++++++++------ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index e803bcb98e6..cda45693a10 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -66,6 +66,11 @@ class MalformedId( MessageException ): err_code = error_codes.MALFORMED_ID +class MalformedContents( MessageException ): + status_code = 400 + err_code = error_codes.MALFORMED_CONTENTS + + class UnknownContentsType( MessageException ): status_code = 400 err_code = error_codes.UNKNOWN_CONTENTS_TYPE diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 2ff18ad50d1..3b203fb1c85 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -56,9 +56,14 @@ }, { "name": "USER_TOOL_META_PARAMETER_PROBLEM", - "code": 400011, + "code": 400012, "message": "Supplied incorrect or incompatible tool meta parameters." }, + { + "name": "MALFORMED_CONTENTS", + "code": 400013, + "message": "The contents of the request are malformed." + }, { "name": "USER_AUTHENTICATION_FAILED", "code": 401001, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 1983bb2f58b..9ab14140640 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,6 +5,7 @@ import tempfile import json import datetime from galaxy import model +from galaxy.exceptions import MalformedContents from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode from galaxy.model.item_attrs import UsesAnnotations @@ -187,9 +188,9 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if dataset_attrs.get('exported', True) == True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ - os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) + os.path.realpath( os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) ) if not file_in_dir( temp_dataset_file_name, os.path.join( archive_dir, "datasets" ) ): - raise Exception( "Invalid dataset path: %s" % temp_dataset_file_name ) + raise MalformedContents( "Invalid dataset path: %s" % temp_dataset_file_name ) if datasets_usage_counts[ temp_dataset_file_name ] == 1: shutil.move( temp_dataset_file_name, hda.file_name ) else: @@ -313,6 +314,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + raise class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index d6890257c8c..f3dab0c5ae4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,6 +6,7 @@ usage: %prog archive_source dest_dir --[url|file] source type, either a URL or a file. """ +import os import sys import optparse import tarfile @@ -43,6 +44,22 @@ def url_to_file( url, dest_file ): return None +def check_archive( archive_file, dest_dir ): + """ + Ensure that a tar archive has no absolute paths or relative paths outside + the archive. + """ + with tarfile.open( archive_file, mode='r:gz' ) as archive_fp: + for arc_path in archive_fp.getnames(): + assert os.path.normpath( + os.path.join( + dest_dir, + arc_path + ) ).startswith( dest_dir.rstrip(os.sep) + os.sep ), \ + "Archive member would extract outside target directory: %s" % arc_path + return True + + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -51,13 +68,8 @@ def unpack_archive( archive_file, dest_dir ): archive_fp.extractall( path=dest_dir ) archive_fp.close() -if __name__ == "__main__": - # Parse command line. - parser = optparse.OptionParser() - parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) - parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) - (options, args) = parser.parse_args() + +def main(options, args): is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args @@ -66,14 +78,25 @@ if __name__ == "__main__": archive_source = b64decode( archive_source ) dest_dir = b64decode( dest_dir ) - try: - # Get archive from URL. - if is_url: - archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) - elif is_file: - archive_file = archive_source + # Get archive from URL. + if is_url: + archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) + elif is_file: + archive_file = archive_source - # Unpack archive. - unpack_archive( archive_file, dest_dir ) + # Unpack archive. + check_archive( archive_file, dest_dir ) + unpack_archive( archive_file, dest_dir ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) + parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) + parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) + (options, args) = parser.parse_args() + try: + main(options, args) except Exception, e: print "Error unpacking tar/gz archive: %s" % e, sys.stderr From d650055ca824d5cf8ab1ede389e9b3060c1f4094 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 05/35] Security fixes for object store paths --- lib/galaxy/objectstore/__init__.py | 21 ++++++++++++++++----- lib/galaxy/objectstore/rods.py | 17 ++++++++++++++++- lib/galaxy/objectstore/s3.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 528193e8f81..2df41134b1c 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,7 +11,7 @@ import logging import threading from xml.etree import ElementTree -from galaxy.util import umask_fix_perms, force_symlink +from galaxy.util import umask_fix_perms, force_symlink, safe_relpath from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util.sleeper import Sleeper from galaxy.util.directory_hash import directory_hash_id @@ -256,7 +256,17 @@ class DiskObjectStore(ObjectStore): the composed directory structure does not include a hash id (e.g., /files/dataset_10.dat (old) vs. /files/000/dataset_10.dat (new)) """ - base = self.extra_dirs.get(base_dir, self.file_path) + base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path)) + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name and not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") if old_style: if extra_dir is not None: path = os.path.join(base, extra_dir) @@ -623,9 +633,10 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 'irods': from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'pulsar': - from .pulsar import PulsarObjectStore - return PulsarObjectStore(config=config, config_xml=config_xml) + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4b814d55d0e..c29161d5326 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,7 +12,8 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.exceptions import ObjectNotFound +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import safe_relpath from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs try: @@ -71,6 +72,20 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but iRODS will + # not follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) path = "" if extra_dir is not None: path = extra_dir diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index a6b97ab4668..a3701ba7fed 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -12,8 +12,8 @@ import time from datetime import datetime -from galaxy.exceptions import ObjectNotFound -from galaxy.util import umask_fix_perms +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import umask_fix_perms, safe_relpath from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sleeper import Sleeper from .s3_multipart_upload import multipart_upload @@ -189,6 +189,20 @@ class S3ObjectStore(ObjectStore): umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) rel_path = os.path.join(*directory_hash_id(obj.id)) if extra_dir is not None: if extra_dir_at_root: From 32c910ac4021b0e65112d557a7f12d24e158171d Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 06/35] Security fixes for object store paths --- lib/galaxy/objectstore/__init__.py | 21 ++++++++++++++++----- lib/galaxy/objectstore/rods.py | 17 ++++++++++++++++- lib/galaxy/objectstore/s3.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 528193e8f81..2df41134b1c 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,7 +11,7 @@ import logging import threading from xml.etree import ElementTree -from galaxy.util import umask_fix_perms, force_symlink +from galaxy.util import umask_fix_perms, force_symlink, safe_relpath from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util.sleeper import Sleeper from galaxy.util.directory_hash import directory_hash_id @@ -256,7 +256,17 @@ class DiskObjectStore(ObjectStore): the composed directory structure does not include a hash id (e.g., /files/dataset_10.dat (old) vs. /files/000/dataset_10.dat (new)) """ - base = self.extra_dirs.get(base_dir, self.file_path) + base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path)) + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name and not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") if old_style: if extra_dir is not None: path = os.path.join(base, extra_dir) @@ -623,9 +633,10 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 'irods': from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'pulsar': - from .pulsar import PulsarObjectStore - return PulsarObjectStore(config=config, config_xml=config_xml) + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4b814d55d0e..c29161d5326 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,7 +12,8 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.exceptions import ObjectNotFound +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import safe_relpath from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs try: @@ -71,6 +72,20 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but iRODS will + # not follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) path = "" if extra_dir is not None: path = extra_dir diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 900713f0b82..dbb3e551a8c 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -12,8 +12,8 @@ import time from datetime import datetime -from galaxy.exceptions import ObjectNotFound -from galaxy.util import umask_fix_perms +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import umask_fix_perms, safe_relpath from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sleeper import Sleeper from .s3_multipart_upload import multipart_upload @@ -181,6 +181,20 @@ class S3ObjectStore(ObjectStore): umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) rel_path = os.path.join(*directory_hash_id(obj.id)) if extra_dir is not None: if extra_dir_at_root: From 37548ef8edb95d59fddd73283561e8b1a6588c10 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 07/35] Remove sample tracking manual external service transfer due to security concerns --- lib/galaxy/model/__init__.py | 25 -- .../galaxy/controllers/requests_admin.py | 296 +----------------- .../requests/select_datasets_to_transfer.mako | 149 --------- .../galaxy/requests/common/common.mako | 12 +- .../requests/common/view_request_history.mako | 1 - .../requests/common/view_sample_datasets.mako | 4 - 6 files changed, 4 insertions(+), 483 deletions(-) delete mode 100644 templates/admin/requests/select_datasets_to_transfer.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6b9a446ae1e..0cbae390ea4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,15 +5,11 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -from galaxy import eggs -eggs.require("pexpect") - import codecs import errno import logging import operator import os -import pexpect import json import socket import time @@ -3858,27 +3854,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password']+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 10665404401..3f75afd54b6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -6,10 +6,7 @@ from galaxy.model.orm import * from galaxy import model, util from galaxy.web.form_builder import * from .requests_common import RequestsGrid, invalid_id_redirect -from galaxy import eggs -eggs.require("amqp") -import amqp -import logging, os, pexpect, ConfigParser +import logging, os, ConfigParser log = logging.getLogger( __name__ ) @@ -351,182 +348,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - files = [] - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
' ) ) - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names def __rename_dataset( self, sample, filepath, scp_configs ): name = filepath.split( '/' )[-1] options = sample.request.type.rename_dataset_options @@ -561,90 +382,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): flush_needed = True if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if not dataset_elements.has_key( external_service ): - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) \ - or not scp_configs.get( 'user_name', '' ) \ - or not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -686,35 +423,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
%s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index 10d2135b7ef..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,149 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

- - -%if not sample: -
- Select a sample before selecting datasets to transfer -
-%endif - -%if request.samples_without_library_destinations: -

- Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

-%endif - -%if message: - ${render_msg( message, status )} -%endif - -
-
Select datasets to transfer from data directory configured for the external service
-
-
- - ${sample_id_select_field.get_html()} -
- Select the sample that was sequenced to produce the datasets you want to transfer. -
-
-
- -
- Loading... -
- -
-
    -
  • Click the external service configuration button and change the Data directory setting to redefine the source data location.
    • -
  • Select a folder to select all of the individual files within that folder.
    • -
  • Click the Select datasets button when desired dataset check boxes are checked.
    • -
-
-
-
-
-
-
- -
-
-
- -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

- ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

- %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request
    • From 78f441b9c7bfd68f7b3ee030f4ffe4a95b5fdc58 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 08/35] Remove sample tracking manual external service transfer due to security concerns --- lib/galaxy/model/__init__.py | 25 -- .../galaxy/controllers/requests_admin.py | 296 +----------------- .../requests/select_datasets_to_transfer.mako | 149 --------- .../galaxy/requests/common/common.mako | 12 +- .../requests/common/view_request_history.mako | 1 - .../requests/common/view_sample_datasets.mako | 4 - 6 files changed, 4 insertions(+), 483 deletions(-) delete mode 100644 templates/admin/requests/select_datasets_to_transfer.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0130033eb41..137d82b1941 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,15 +5,11 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -from galaxy import eggs -eggs.require("pexpect") - import codecs import errno import logging import operator import os -import pexpect import json import socket import time @@ -3694,27 +3690,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password']+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 10665404401..3f75afd54b6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -6,10 +6,7 @@ from galaxy.model.orm import * from galaxy import model, util from galaxy.web.form_builder import * from .requests_common import RequestsGrid, invalid_id_redirect -from galaxy import eggs -eggs.require("amqp") -import amqp -import logging, os, pexpect, ConfigParser +import logging, os, ConfigParser log = logging.getLogger( __name__ ) @@ -351,182 +348,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - files = [] - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
    ' ) ) - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names def __rename_dataset( self, sample, filepath, scp_configs ): name = filepath.split( '/' )[-1] options = sample.request.type.rename_dataset_options @@ -561,90 +382,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): flush_needed = True if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if not dataset_elements.has_key( external_service ): - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) \ - or not scp_configs.get( 'user_name', '' ) \ - or not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -686,35 +423,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
    %s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index 10d2135b7ef..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,149 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

    - - -%if not sample: -
    - Select a sample before selecting datasets to transfer -
    -%endif - -%if request.samples_without_library_destinations: -

    - Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

    -%endif - -%if message: - ${render_msg( message, status )} -%endif - -
    -
    Select datasets to transfer from data directory configured for the external service
    -
    -
    - - ${sample_id_select_field.get_html()} -
    - Select the sample that was sequenced to produce the datasets you want to transfer. -
    -
    -
    - -
    - Loading... -
    - -
    -
      -
    • Click the external service configuration button and change the Data directory setting to redefine the source data location.
      • -
    • Select a folder to select all of the individual files within that folder.
      • -
    • Click the Select datasets button when desired dataset check boxes are checked.
      • -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    - -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

    - ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

    - %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request
    • From 4f3397fa0ab187a4b5c32e1268265925fc1a8753 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 09/35] Security fixes for tool shed repository browsing --- .../galaxy/controllers/admin_toolshed.py | 8 ++-- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/util/shed_util_common.py | 46 +++++++++++++++---- .../admin/tool_shed_repository/common.mako | 12 +++-- .../webapps/tool_shed/repository/common.mako | 10 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index c7ff192ad27..9866f699e35 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -383,11 +383,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose @web.require_admin @@ -960,11 +960,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 25f15b36240..7493513a2b3 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1629,11 +1629,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose def get_functional_test_rss( self, trans, **kwd ): @@ -2603,11 +2603,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index be8ef343ffc..4356bc0e738 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -509,9 +509,17 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha changeset_revision=text ) return repository -def get_repository_file_contents( file_path ): +def get_repository_file_contents( app, file_path, repository_id ): """Return the display-safe contents of a repository file for display in a browser.""" - if checkers.is_gzip( file_path ): + safe_str = '' + if not is_path_within_repo( app, file_path, repository_id ): + log.warning( 'Request tries to access a file outside of the repository location. File path: %s', file_path ) + return 'Invalid file path' + # Symlink targets are checked by is_path_within_repo + if os.path.islink( file_path ): + safe_str = 'link to: ' + basic_util.to_html_string( os.readlink( file_path ) ) + return safe_str + elif checkers.is_gzip( file_path ): return '
    gzip compressed file
    ' elif checkers.is_bz2( file_path ): return '
    bz2 compressed file
    ' @@ -520,7 +528,6 @@ def get_repository_file_contents( file_path ): elif checkers.check_binary( file_path ): return '
    Binary file
    ' else: - safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. @@ -530,6 +537,7 @@ def get_repository_file_contents( file_path ): util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. @@ -550,9 +558,6 @@ def get_repository_files( folder_path ): # Skip .hg directories if item.startswith( '.hg' ): continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item contents.append( item ) if contents: contents.sort() @@ -1008,11 +1013,15 @@ def is_tool_shed_client( app ): """ return hasattr( app, "install_model" ) -def open_repository_files_folder( folder_path ): +def open_repository_files_folder( app, folder_path, repository_id ): """ Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy. """ + # Symlink targets are checked by is_path_within_repo + if not is_path_within_repo( app, folder_path, repository_id ): + log.warning( 'Request tries to access a folder outside of the repository location. Folder path: %s', folder_path ) + return [] try: files_list = get_repository_files( folder_path ) except OSError, e: @@ -1022,10 +1031,17 @@ def open_repository_files_folder( folder_path ): folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[ -1 ] == os.sep: - is_folder = True + full_path = os.path.join( folder_path, filename ) + is_link = os.path.islink( full_path ) + path_is_within_repo = is_path_within_repo( app, full_path, repository_id ) + if is_link and not path_is_within_repo: + log.warning( 'Valid folder contains a symlink outside of the repository location. Link found in: ' + str( full_path ) ) if filename: - full_path = os.path.join( folder_path, filename ) + if os.path.isdir( full_path ) and path_is_within_repo: + # Append a '/' character so that our jquery dynatree will function properly. + filename = '%s/' % filename + full_path = '%s/' % full_path + is_folder = True node = { "title" : filename, "isFolder" : is_folder, "isLazy" : is_folder, @@ -1034,6 +1050,16 @@ def open_repository_files_folder( folder_path ): folder_contents.append( node ) return folder_contents +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folde ron the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path + + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 1fb124270c9..bea2d78e5d7 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -23,14 +23,16 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${directory_path|h}" }, + dataType: "json", + data: { folder_path: "${directory_path|h}", + repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", + url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, + repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -55,7 +57,7 @@ type: "POST", url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function( data ) { cell.html( '' ) } diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index a18fd700149..78ca70ae670 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -21,14 +21,14 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='repository', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${repository.repo_path( trans.app )}" }, + dataType: "json", + data: { folder_path: "${repository.repo_path( trans.app )}", repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='repository', action='open_folder' )}", + url: "${h.url_for( controller='repository', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -61,7 +61,7 @@ type: "POST", url: "${h.url_for( controller='repository', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function ( data ) { cell.html( '' ) } From 8468871ee481598a84888b5329139590ec75e98d Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 10/35] Add a safe_relpath util function for ensuring a path does not reference an absolute or parent directory --- lib/galaxy/util/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 9d956aca92f..12131cde63f 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -28,7 +28,7 @@ from datetime import datetime from email.MIMEText import MIMEText -from os.path import relpath +from os.path import relpath, normpath from hashlib import md5 from itertools import izip @@ -1243,6 +1243,23 @@ galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") def galaxy_directory(): return os.path.abspath(galaxy_root_path) + +def safe_relpath(path): + """ + Given what we expect to be a relative path, determine whether the path + would exist inside the current directory. + + :type path: string + :param path: a path to check + :rtype: bool + :returns: ``True`` if path is relative and does not reference a path + in a parent directory, ``False`` otherwise. + """ + if path.startswith(os.sep) or normpath(path).startswith(os.pardir): + return False + return True + + if __name__ == '__main__': import doctest doctest.testmod(sys.modules[__name__], verbose=False) From cee85bae67377d3fad42bec5b4ebb022aaf7b2c7 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 11/35] Security fixes for tool shed repository browsing --- .../galaxy/controllers/admin_toolshed.py | 8 ++-- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/util/shed_util_common.py | 46 +++++++++++++++---- .../admin/tool_shed_repository/common.mako | 12 +++-- .../webapps/tool_shed/repository/common.mako | 10 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index b31177fabd6..1be3a2a17c3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -383,11 +383,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose @web.require_admin @@ -960,11 +960,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 9d8bc5ab660..b2a35250628 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1629,11 +1629,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose def get_functional_test_rss( self, trans, **kwd ): @@ -2600,11 +2600,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index be8ef343ffc..4356bc0e738 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -509,9 +509,17 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha changeset_revision=text ) return repository -def get_repository_file_contents( file_path ): +def get_repository_file_contents( app, file_path, repository_id ): """Return the display-safe contents of a repository file for display in a browser.""" - if checkers.is_gzip( file_path ): + safe_str = '' + if not is_path_within_repo( app, file_path, repository_id ): + log.warning( 'Request tries to access a file outside of the repository location. File path: %s', file_path ) + return 'Invalid file path' + # Symlink targets are checked by is_path_within_repo + if os.path.islink( file_path ): + safe_str = 'link to: ' + basic_util.to_html_string( os.readlink( file_path ) ) + return safe_str + elif checkers.is_gzip( file_path ): return '
    gzip compressed file
    ' elif checkers.is_bz2( file_path ): return '
    bz2 compressed file
    ' @@ -520,7 +528,6 @@ def get_repository_file_contents( file_path ): elif checkers.check_binary( file_path ): return '
    Binary file
    ' else: - safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. @@ -530,6 +537,7 @@ def get_repository_file_contents( file_path ): util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. @@ -550,9 +558,6 @@ def get_repository_files( folder_path ): # Skip .hg directories if item.startswith( '.hg' ): continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item contents.append( item ) if contents: contents.sort() @@ -1008,11 +1013,15 @@ def is_tool_shed_client( app ): """ return hasattr( app, "install_model" ) -def open_repository_files_folder( folder_path ): +def open_repository_files_folder( app, folder_path, repository_id ): """ Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy. """ + # Symlink targets are checked by is_path_within_repo + if not is_path_within_repo( app, folder_path, repository_id ): + log.warning( 'Request tries to access a folder outside of the repository location. Folder path: %s', folder_path ) + return [] try: files_list = get_repository_files( folder_path ) except OSError, e: @@ -1022,10 +1031,17 @@ def open_repository_files_folder( folder_path ): folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[ -1 ] == os.sep: - is_folder = True + full_path = os.path.join( folder_path, filename ) + is_link = os.path.islink( full_path ) + path_is_within_repo = is_path_within_repo( app, full_path, repository_id ) + if is_link and not path_is_within_repo: + log.warning( 'Valid folder contains a symlink outside of the repository location. Link found in: ' + str( full_path ) ) if filename: - full_path = os.path.join( folder_path, filename ) + if os.path.isdir( full_path ) and path_is_within_repo: + # Append a '/' character so that our jquery dynatree will function properly. + filename = '%s/' % filename + full_path = '%s/' % full_path + is_folder = True node = { "title" : filename, "isFolder" : is_folder, "isLazy" : is_folder, @@ -1034,6 +1050,16 @@ def open_repository_files_folder( folder_path ): folder_contents.append( node ) return folder_contents +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folde ron the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path + + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 1fb124270c9..bea2d78e5d7 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -23,14 +23,16 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${directory_path|h}" }, + dataType: "json", + data: { folder_path: "${directory_path|h}", + repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", + url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, + repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -55,7 +57,7 @@ type: "POST", url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function( data ) { cell.html( '' ) } diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index d7a6fe865d9..af7be14856e 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -21,14 +21,14 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='repository', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${repository.repo_path( trans.app )}" }, + dataType: "json", + data: { folder_path: "${repository.repo_path( trans.app )}", repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='repository', action='open_folder' )}", + url: "${h.url_for( controller='repository', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -61,7 +61,7 @@ type: "POST", url: "${h.url_for( controller='repository', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function ( data ) { cell.html( '' ) } From 4845a39071ab4e2652e564efdf7647a942563603 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 12/35] Security fixes for tool shed hg push and capsule/tarball uploads --- .../tool_shed/framework/middleware/hg.py | 21 ++++++- lib/tool_shed/capsule/capsule_manager.py | 57 ++++++++++++------- lib/tool_shed/util/commit_util.py | 54 +++++++++++------- lib/tool_shed/util/repository_content_util.py | 26 +++------ 4 files changed, 99 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 14745cb6446..f07f017837c 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -9,7 +9,7 @@ from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import AUTH_TYPE from paste.httpheaders import REMOTE_USER -from galaxy.util import asbool +from galaxy.util import asbool, safe_relpath from galaxy.util.hash_util import new_secure_hash from tool_shed.util import hg_util import tool_shed.repository_types.util as rt_util @@ -113,7 +113,11 @@ class Hg( object ): fh.write( chunk ) fh.close() fh = open( tmp_filename, 'rb' ) - changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + try: + changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + except AttributeError: + msg = 'Your version of Mercurial is not supported. Please use a version < 3.5' + return self.__display_exception_remotely( start_response, msg ) fh.close() try: os.unlink( tmp_filename ) @@ -122,6 +126,19 @@ class Hg( object ): if changeset_groups: # Check the repository type to make sure inappropriate files are not being pushed. if 'PATH_INFO' in environ: + # Ensure there are no symlinks with targets outside the repo + for entry in changeset_groups: + if len( entry ) == 2: + filename, change_list = entry + if not isinstance(change_list, list): + change_list = [change_list] + for change in change_list: + for patch in change['data']: + target = patch['block'].strip() + if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ): + msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target ) + log.warning( msg ) + return self.__display_exception_remotely( start_response, msg ) # Instantiate a database connection engine = sqlalchemy.create_engine( self.db_url ) connection = engine.connect() diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 15b6fb43d87..642b04f33d9 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -12,6 +12,7 @@ from galaxy import web from galaxy.model.orm import and_ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE +from galaxy.util import safe_relpath from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder @@ -727,29 +728,16 @@ class ImportRepositoryManager( object ): repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, error_message = commit_util.check_archive( repository, archive ) - if ok: + check_results = commit_util.check_archive( repository, archive ) + # We filter out undesirable files but fail on undesriable dirs. Not + # sure why, just trying to maintain the same behavior as before. -nate + if not check_results.invalid and not check_results.undesirable_dirs: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in archive.getmembers(): - # Check files and directories in the archive. - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - error_message = 'Import failed: invalid file path %s in archive %s' % \ - ( str( file_path_item ), str( archive_file_name ) ) - results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message - return results_dict - filenames_in_archive.append( tarinfo_obj.name ) - else: - undesirable_files_removed += 1 # Extract the uploaded archive to the repository root. - archive.extractall( path=full_path ) + archive.extractall( path=full_path, members=check_results.valid ) archive.close() - for filename in filenames_in_archive: + for tar_member in check_results.valid: + filename = tar_member.name uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if toolshed or changeset_revision attributes @@ -776,6 +764,9 @@ class ImportRepositoryManager( object ): new_repo_alert = True # Since the repository is new, the following must be False. remove_repo_files_not_in_tar = False + filenames_in_archive = [ member.name for member in check_results.valid ] + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = 0 ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ commit_util.handle_directory_changes( self.app, self.host, @@ -806,7 +797,13 @@ class ImportRepositoryManager( object ): else: archive.close() results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message + results_dict[ 'error_message' ] += 'Capsule errors were found: ' + if check_results.invalid: + results_dict[ 'error_message' ] += '%s Invalid files were: %s.' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + if check_results.undesirable_dirs: + results_dict[ 'error_message' ] += ' Undesirable directories were: %s.' % ( + ', '.join( check_results.undesirable_dirs ) ) return results_dict def upload_capsule( self, **kwd ): @@ -863,6 +860,12 @@ class ImportRepositoryManager( object ): return_dict[ 'status' ] = 'error' uploaded_file.close() return return_dict + if not self.validate_archive_paths( tar_archive ): + return_dict[ 'status' ] = 'error' + return_dict[ 'message' ] = ( 'This capsule contains an invalid member type ' + 'or a file outside the archive path.' ) + uploaded_file.close() + return return_dict return_dict[ 'tar_archive' ] = tar_archive return_dict[ 'capsule_file_name' ] = uploaded_file_filename uploaded_file.close() @@ -872,6 +875,18 @@ class ImportRepositoryManager( object ): return return_dict return return_dict + def validate_archive_paths( self, tar_archive ): + ''' + Inspect the archive contents to ensure that there are no risky symlinks. + Returns True if a suspicious path is found. + ''' + for member in tar_archive.getmembers(): + if not ( member.isdir() or member.isfile() or member.islnk() ): + return False + elif not safe_relpath( member.name ): + return False + return True + def validate_capsule( self, **kwd ): """ Inspect the uploaded capsule's manifest and its contained files to ensure it is a valid diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 4b8d9680f68..cfefbb79ae2 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -4,8 +4,10 @@ import logging import os import shutil import tempfile +from collections import namedtuple from galaxy.datatypes import checkers +from galaxy.util import safe_relpath from tool_shed.tools import data_table_manager @@ -21,30 +23,44 @@ UNDESIRABLE_DIRS = [ '.hg', '.svn', '.git', '.cvs' ] UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store' ] def check_archive( repository, archive ): + valid = [] + invalid = [] + errors = [] + undesirable_files = [] + undesirable_dirs = [] for member in archive.getmembers(): # Allow regular files and directories only if not ( member.isdir() or member.isfile() or member.islnk() ): - message = "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc). " - message += "The problematic member in this archive is %s," % str( member.name ) - return False, message - for item in [ '.hg', '..', '/' ]: - if member.name.startswith( item ): - message = "Uploaded archives cannot contain .hg directories, absolute filenames starting with '/', or filenames with two dots '..'. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message - if member.name in [ 'hgrc' ]: - message = "Uploaded archives cannot contain hgrc files. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message + errors.append( "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc)." ) + invalid.append( member ) + continue + if not safe_relpath( member.name ): + errors.append( "Uploaded archives cannot contain files that would extract outside of the archive." ) + invalid.append( member ) + continue + if os.path.basename( member.name ) in UNDESIRABLE_FILES: + undesirable_files.append( member ) + continue + head = tail = member.name + try: + while tail: + head, tail = os.path.split(head) + if tail in UNDESIRABLE_DIRS: + undesirable_dirs.append( member ) + assert False + except AssertionError: + continue if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message + errors.append( 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' ) + invalid.append( member ) + continue if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message - return True, '' + errors.append( 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' ) + invalid.append( member ) + continue + valid.append( member ) + ArchiveCheckResults = namedtuple( 'ArchiveCheckResults', [ 'valid', 'invalid', 'undesirable_files', 'undesirable_dirs', 'errors' ] ) + return ArchiveCheckResults( valid, invalid, undesirable_files, undesirable_dirs, errors ) def check_file_contents_for_email_alerts( app ): """ diff --git a/lib/tool_shed/util/repository_content_util.py b/lib/tool_shed/util/repository_content_util.py index 99fc9793b83..1cf6660278c 100644 --- a/lib/tool_shed/util/repository_content_util.py +++ b/lib/tool_shed/util/repository_content_util.py @@ -15,31 +15,23 @@ def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, message = commit_util.check_archive( repository, tar ) - if not ok: + check_results = commit_util.check_archive( repository, tar ) + if check_results.invalid: tar.close() uploaded_file.close() - return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed + message = '%s Invalid paths were: %s' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) else: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in tar.getmembers(): - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - ok = False - break - else: - undesirable_files_removed += 1 - if ok: - filenames_in_archive.append( tarinfo_obj.name ) + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = len( check_results.undesirable_dirs ) + filenames_in_archive = [ ti.name for ti in check_results.valid ] # Extract the uploaded tar to the load_point within the repository hierarchy. - tar.extractall( path=full_path ) + tar.extractall( path=full_path, members=check_results.valid ) tar.close() uploaded_file.close() for filename in filenames_in_archive: From ef4a9f6e2114e69f6ea3167e99a857f2ea4d28af Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:19 -0500 Subject: [PATCH 13/35] Security fixes for history imports --- lib/galaxy/exceptions/__init__.py | 5 ++ lib/galaxy/exceptions/error_codes.json | 7 ++- lib/galaxy/tools/imp_exp/__init__.py | 6 ++- .../tools/imp_exp/unpack_tar_gz_archive.py | 53 +++++++++++++------ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index d881fce0a8f..9bce4857152 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -71,6 +71,11 @@ class MalformedId( MessageException ): err_code = error_codes.MALFORMED_ID +class MalformedContents( MessageException ): + status_code = 400 + err_code = error_codes.MALFORMED_CONTENTS + + class UnknownContentsType( MessageException ): status_code = 400 err_code = error_codes.UNKNOWN_CONTENTS_TYPE diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 92c85e2f4d6..db3509b29a1 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -61,9 +61,14 @@ }, { "name": "USER_TOOL_META_PARAMETER_PROBLEM", - "code": 400011, + "code": 400012, "message": "Supplied incorrect or incompatible tool meta parameters." }, + { + "name": "MALFORMED_CONTENTS", + "code": 400013, + "message": "The contents of the request are malformed." + }, { "name": "USER_AUTHENTICATION_FAILED", "code": 401001, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index c9a7f90c93d..536413b511d 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,6 +5,7 @@ import tempfile import json import datetime from galaxy import model +from galaxy.exceptions import MalformedContents from galaxy.model.item_attrs import UsesAnnotations from galaxy.model.orm import eagerload, eagerload_all from galaxy.tools.parameters.basic import UnvalidatedValue @@ -190,9 +191,9 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): if dataset_attrs.get('exported', True) is True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ - os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) + os.path.realpath( os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) ) if not file_in_dir( temp_dataset_file_name, os.path.join( archive_dir, "datasets" ) ): - raise Exception( "Invalid dataset path: %s" % temp_dataset_file_name ) + raise MalformedContents( "Invalid dataset path: %s" % temp_dataset_file_name ) if datasets_usage_counts[ temp_dataset_file_name ] == 1: shutil.move( temp_dataset_file_name, hda.file_name ) else: @@ -314,6 +315,7 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + raise class JobExportHistoryArchiveWrapper( object, UsesAnnotations ): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index d6890257c8c..f3dab0c5ae4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,6 +6,7 @@ usage: %prog archive_source dest_dir --[url|file] source type, either a URL or a file. """ +import os import sys import optparse import tarfile @@ -43,6 +44,22 @@ def url_to_file( url, dest_file ): return None +def check_archive( archive_file, dest_dir ): + """ + Ensure that a tar archive has no absolute paths or relative paths outside + the archive. + """ + with tarfile.open( archive_file, mode='r:gz' ) as archive_fp: + for arc_path in archive_fp.getnames(): + assert os.path.normpath( + os.path.join( + dest_dir, + arc_path + ) ).startswith( dest_dir.rstrip(os.sep) + os.sep ), \ + "Archive member would extract outside target directory: %s" % arc_path + return True + + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -51,13 +68,8 @@ def unpack_archive( archive_file, dest_dir ): archive_fp.extractall( path=dest_dir ) archive_fp.close() -if __name__ == "__main__": - # Parse command line. - parser = optparse.OptionParser() - parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) - parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) - (options, args) = parser.parse_args() + +def main(options, args): is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args @@ -66,14 +78,25 @@ if __name__ == "__main__": archive_source = b64decode( archive_source ) dest_dir = b64decode( dest_dir ) - try: - # Get archive from URL. - if is_url: - archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) - elif is_file: - archive_file = archive_source + # Get archive from URL. + if is_url: + archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) + elif is_file: + archive_file = archive_source - # Unpack archive. - unpack_archive( archive_file, dest_dir ) + # Unpack archive. + check_archive( archive_file, dest_dir ) + unpack_archive( archive_file, dest_dir ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) + parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) + parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) + (options, args) = parser.parse_args() + try: + main(options, args) except Exception, e: print "Error unpacking tar/gz archive: %s" % e, sys.stderr From 2a8089749caab12c087064c840708fb05b0104cd Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 14/35] Add a safe_relpath util function for ensuring a path does not reference an absolute or parent directory --- lib/galaxy/util/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 39f84532aee..f17362769ca 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -29,7 +29,7 @@ from datetime import datetime from email.MIMEText import MIMEText -from os.path import relpath +from os.path import relpath, normpath from hashlib import md5 from itertools import izip @@ -1252,6 +1252,22 @@ def galaxy_directory(): return os.path.abspath(galaxy_root_path) +def safe_relpath(path): + """ + Given what we expect to be a relative path, determine whether the path + would exist inside the current directory. + + :type path: string + :param path: a path to check + :rtype: bool + :returns: ``True`` if path is relative and does not reference a path + in a parent directory, ``False`` otherwise. + """ + if path.startswith(os.sep) or normpath(path).startswith(os.pardir): + return False + return True + + class ExecutionTimer(object): def __init__(self): From c8963786d53efd2cf3b359b2f9a000e00688285f Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 15/35] Security fixes for object store paths --- lib/galaxy/objectstore/__init__.py | 21 ++++++++++++++++----- lib/galaxy/objectstore/rods.py | 17 ++++++++++++++++- lib/galaxy/objectstore/s3.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 7d6d959b3fd..55a120b94db 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,7 +11,7 @@ import logging import threading from xml.etree import ElementTree -from galaxy.util import umask_fix_perms, force_symlink +from galaxy.util import umask_fix_perms, force_symlink, safe_relpath from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util.sleeper import Sleeper from galaxy.util.directory_hash import directory_hash_id @@ -256,7 +256,17 @@ class DiskObjectStore(ObjectStore): the composed directory structure does not include a hash id (e.g., /files/dataset_10.dat (old) vs. /files/000/dataset_10.dat (new)) """ - base = self.extra_dirs.get(base_dir, self.file_path) + base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path)) + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name and not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") if old_style: if extra_dir is not None: path = os.path.join(base, extra_dir) @@ -623,9 +633,10 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 'irods': from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'pulsar': - from .pulsar import PulsarObjectStore - return PulsarObjectStore(config=config, config_xml=config_xml) + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4b814d55d0e..c29161d5326 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,7 +12,8 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.exceptions import ObjectNotFound +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import safe_relpath from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs try: @@ -71,6 +72,20 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but iRODS will + # not follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) path = "" if extra_dir is not None: path = extra_dir diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 361c40a915a..a908e66a4ad 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -12,8 +12,8 @@ import time from datetime import datetime -from galaxy.exceptions import ObjectNotFound -from galaxy.util import string_as_bool, umask_fix_perms +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import string_as_bool, umask_fix_perms, safe_relpath from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sleeper import Sleeper from .s3_multipart_upload import multipart_upload @@ -201,6 +201,20 @@ class S3ObjectStore(ObjectStore): umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) rel_path = os.path.join(*directory_hash_id(obj.id)) if extra_dir is not None: if extra_dir_at_root: From 6e3b74869a54558fe48d6e1633c5af7197051447 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 16/35] Security fixes for history imports --- lib/galaxy/exceptions/__init__.py | 5 ++ lib/galaxy/exceptions/error_codes.json | 7 ++- lib/galaxy/tools/imp_exp/__init__.py | 6 ++- .../tools/imp_exp/unpack_tar_gz_archive.py | 53 +++++++++++++------ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 65210229d55..4421d994e76 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -71,6 +71,11 @@ class MalformedId( MessageException ): err_code = error_codes.MALFORMED_ID +class MalformedContents( MessageException ): + status_code = 400 + err_code = error_codes.MALFORMED_CONTENTS + + class UnknownContentsType( MessageException ): status_code = 400 err_code = error_codes.UNKNOWN_CONTENTS_TYPE diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index c681afdebb0..50faf0fc644 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -61,9 +61,14 @@ }, { "name": "USER_TOOL_META_PARAMETER_PROBLEM", - "code": 400011, + "code": 400012, "message": "Supplied incorrect or incompatible tool meta parameters." }, + { + "name": "MALFORMED_CONTENTS", + "code": 400013, + "message": "The contents of the request are malformed." + }, { "name": "USER_AUTHENTICATION_FAILED", "code": 401001, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 364e9351179..f5a5c7ddb2b 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,6 +5,7 @@ import tempfile import json import datetime from galaxy import model +from galaxy.exceptions import MalformedContents from galaxy.model.item_attrs import UsesAnnotations from galaxy.model.orm import eagerload, eagerload_all from galaxy.tools.parameters.basic import UnvalidatedValue @@ -192,9 +193,9 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): if dataset_attrs.get('exported', True) is True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ - os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) + os.path.realpath( os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) ) if not file_in_dir( temp_dataset_file_name, os.path.join( archive_dir, "datasets" ) ): - raise Exception( "Invalid dataset path: %s" % temp_dataset_file_name ) + raise MalformedContents( "Invalid dataset path: %s" % temp_dataset_file_name ) if datasets_usage_counts[ temp_dataset_file_name ] == 1: shutil.move( temp_dataset_file_name, hda.file_name ) else: @@ -316,6 +317,7 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + raise class JobExportHistoryArchiveWrapper( object, UsesAnnotations ): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index d6890257c8c..f3dab0c5ae4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,6 +6,7 @@ usage: %prog archive_source dest_dir --[url|file] source type, either a URL or a file. """ +import os import sys import optparse import tarfile @@ -43,6 +44,22 @@ def url_to_file( url, dest_file ): return None +def check_archive( archive_file, dest_dir ): + """ + Ensure that a tar archive has no absolute paths or relative paths outside + the archive. + """ + with tarfile.open( archive_file, mode='r:gz' ) as archive_fp: + for arc_path in archive_fp.getnames(): + assert os.path.normpath( + os.path.join( + dest_dir, + arc_path + ) ).startswith( dest_dir.rstrip(os.sep) + os.sep ), \ + "Archive member would extract outside target directory: %s" % arc_path + return True + + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -51,13 +68,8 @@ def unpack_archive( archive_file, dest_dir ): archive_fp.extractall( path=dest_dir ) archive_fp.close() -if __name__ == "__main__": - # Parse command line. - parser = optparse.OptionParser() - parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) - parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) - (options, args) = parser.parse_args() + +def main(options, args): is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args @@ -66,14 +78,25 @@ if __name__ == "__main__": archive_source = b64decode( archive_source ) dest_dir = b64decode( dest_dir ) - try: - # Get archive from URL. - if is_url: - archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) - elif is_file: - archive_file = archive_source + # Get archive from URL. + if is_url: + archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) + elif is_file: + archive_file = archive_source - # Unpack archive. - unpack_archive( archive_file, dest_dir ) + # Unpack archive. + check_archive( archive_file, dest_dir ) + unpack_archive( archive_file, dest_dir ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) + parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) + parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) + (options, args) = parser.parse_args() + try: + main(options, args) except Exception, e: print "Error unpacking tar/gz archive: %s" % e, sys.stderr From d4b94722f992c9309aa2ec5bfa5bce2dacd9b218 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 17/35] Remove sample tracking manual external service transfer due to security concerns --- lib/galaxy/model/__init__.py | 25 -- .../galaxy/controllers/requests_admin.py | 296 +----------------- .../requests/select_datasets_to_transfer.mako | 149 --------- .../galaxy/requests/common/common.mako | 12 +- .../requests/common/view_request_history.mako | 1 - .../requests/common/view_sample_datasets.mako | 4 - 6 files changed, 4 insertions(+), 483 deletions(-) delete mode 100644 templates/admin/requests/select_datasets_to_transfer.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4364a6703fe..560b986058c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,15 +5,11 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -from galaxy import eggs -eggs.require("pexpect") - import codecs import errno import logging import operator import os -import pexpect import json import socket import time @@ -3937,27 +3933,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password']+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 10665404401..3f75afd54b6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -6,10 +6,7 @@ from galaxy.model.orm import * from galaxy import model, util from galaxy.web.form_builder import * from .requests_common import RequestsGrid, invalid_id_redirect -from galaxy import eggs -eggs.require("amqp") -import amqp -import logging, os, pexpect, ConfigParser +import logging, os, ConfigParser log = logging.getLogger( __name__ ) @@ -351,182 +348,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - files = [] - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
    ' ) ) - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names def __rename_dataset( self, sample, filepath, scp_configs ): name = filepath.split( '/' )[-1] options = sample.request.type.rename_dataset_options @@ -561,90 +382,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): flush_needed = True if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if not dataset_elements.has_key( external_service ): - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) \ - or not scp_configs.get( 'user_name', '' ) \ - or not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -686,35 +423,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
    %s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index 10d2135b7ef..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,149 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

    - - -%if not sample: -
    - Select a sample before selecting datasets to transfer -
    -%endif - -%if request.samples_without_library_destinations: -

    - Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

    -%endif - -%if message: - ${render_msg( message, status )} -%endif - -
    -
    Select datasets to transfer from data directory configured for the external service
    -
    -
    - - ${sample_id_select_field.get_html()} -
    - Select the sample that was sequenced to produce the datasets you want to transfer. -
    -
    -
    - -
    - Loading... -
    - -
    -
      -
    • Click the external service configuration button and change the Data directory setting to redefine the source data location.
      • -
    • Select a folder to select all of the individual files within that folder.
      • -
    • Click the Select datasets button when desired dataset check boxes are checked.
      • -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    - -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

    - ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

    - %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request
    • From ce88f2f81caff6af7dc730ef9c1fbc2a40922dab Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 18/35] Security fixes for object store paths --- lib/galaxy/objectstore/__init__.py | 21 ++++++++++++++++----- lib/galaxy/objectstore/rods.py | 17 ++++++++++++++++- lib/galaxy/objectstore/s3.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 123d16b846b..949c9d1bfc7 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,7 +11,7 @@ import logging import threading from xml.etree import ElementTree -from galaxy.util import umask_fix_perms, force_symlink +from galaxy.util import umask_fix_perms, force_symlink, safe_relpath from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util.sleeper import Sleeper from galaxy.util.directory_hash import directory_hash_id @@ -252,7 +252,17 @@ class DiskObjectStore(ObjectStore): the composed directory structure does not include a hash id (e.g., /files/dataset_10.dat (old) vs. /files/000/dataset_10.dat (new)) """ - base = self.extra_dirs.get(base_dir, self.file_path) + base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path)) + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name and not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") if old_style: if extra_dir is not None: path = os.path.join(base, extra_dir) @@ -619,9 +629,10 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 'irods': from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'pulsar': - from .pulsar import PulsarObjectStore - return PulsarObjectStore(config=config, config_xml=config_xml) + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4754156ad5b..f4bb03c2706 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,7 +12,8 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.exceptions import ObjectNotFound +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import safe_relpath from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs try: @@ -71,6 +72,20 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but iRODS will + # not follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) path = "" if extra_dir is not None: path = extra_dir diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 361c40a915a..a908e66a4ad 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -12,8 +12,8 @@ import time from datetime import datetime -from galaxy.exceptions import ObjectNotFound -from galaxy.util import string_as_bool, umask_fix_perms +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import string_as_bool, umask_fix_perms, safe_relpath from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sleeper import Sleeper from .s3_multipart_upload import multipart_upload @@ -201,6 +201,20 @@ class S3ObjectStore(ObjectStore): umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) rel_path = os.path.join(*directory_hash_id(obj.id)) if extra_dir is not None: if extra_dir_at_root: From 626cade512894e54c5b3e5257543f267a1826a71 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 19/35] Security fixes for tool shed repository browsing --- .../galaxy/controllers/admin_toolshed.py | 8 ++-- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/util/shed_util_common.py | 46 +++++++++++++++---- .../admin/tool_shed_repository/common.mako | 12 +++-- .../webapps/tool_shed/repository/common.mako | 10 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 095e623deba..b63beba5574 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -384,11 +384,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose @web.require_admin @@ -961,11 +961,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 80819f2236c..3fe52410515 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1635,11 +1635,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose def get_functional_test_rss( self, trans, **kwd ): @@ -2619,11 +2619,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index bf0477995ed..fece59835fc 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -535,9 +535,17 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha return repository -def get_repository_file_contents( file_path ): +def get_repository_file_contents( app, file_path, repository_id ): """Return the display-safe contents of a repository file for display in a browser.""" - if checkers.is_gzip( file_path ): + safe_str = '' + if not is_path_within_repo( app, file_path, repository_id ): + log.warning( 'Request tries to access a file outside of the repository location. File path: %s', file_path ) + return 'Invalid file path' + # Symlink targets are checked by is_path_within_repo + if os.path.islink( file_path ): + safe_str = 'link to: ' + basic_util.to_html_string( os.readlink( file_path ) ) + return safe_str + elif checkers.is_gzip( file_path ): return '
    gzip compressed file
    ' elif checkers.is_bz2( file_path ): return '
    bz2 compressed file
    ' @@ -546,7 +554,6 @@ def get_repository_file_contents( file_path ): elif checkers.check_binary( file_path ): return '
    Binary file
    ' else: - safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. @@ -556,6 +563,7 @@ def get_repository_file_contents( file_path ): util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. @@ -577,9 +585,6 @@ def get_repository_files( folder_path ): # Skip .hg directories if item.startswith( '.hg' ): continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item contents.append( item ) if contents: contents.sort() @@ -1049,11 +1054,15 @@ def is_tool_shed_client( app ): return hasattr( app, "install_model" ) -def open_repository_files_folder( folder_path ): +def open_repository_files_folder( app, folder_path, repository_id ): """ Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy. """ + # Symlink targets are checked by is_path_within_repo + if not is_path_within_repo( app, folder_path, repository_id ): + log.warning( 'Request tries to access a folder outside of the repository location. Folder path: %s', folder_path ) + return [] try: files_list = get_repository_files( folder_path ) except OSError, e: @@ -1063,10 +1072,17 @@ def open_repository_files_folder( folder_path ): folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[ -1 ] == os.sep: - is_folder = True + full_path = os.path.join( folder_path, filename ) + is_link = os.path.islink( full_path ) + path_is_within_repo = is_path_within_repo( app, full_path, repository_id ) + if is_link and not path_is_within_repo: + log.warning( 'Valid folder contains a symlink outside of the repository location. Link found in: ' + str( full_path ) ) if filename: - full_path = os.path.join( folder_path, filename ) + if os.path.isdir( full_path ) and path_is_within_repo: + # Append a '/' character so that our jquery dynatree will function properly. + filename = '%s/' % filename + full_path = '%s/' % full_path + is_folder = True node = { "title": filename, "isFolder": is_folder, "isLazy": is_folder, @@ -1076,6 +1092,16 @@ def open_repository_files_folder( folder_path ): return folder_contents +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folde ron the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path + + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 6cf501e1baa..ec06a65a6c3 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -23,14 +23,16 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${directory_path|h}" }, + dataType: "json", + data: { folder_path: "${directory_path|h}", + repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", + url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, + repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -55,7 +57,7 @@ type: "POST", url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function( data ) { cell.html( '' ) } diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index a18fd700149..78ca70ae670 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -21,14 +21,14 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='repository', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${repository.repo_path( trans.app )}" }, + dataType: "json", + data: { folder_path: "${repository.repo_path( trans.app )}", repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='repository', action='open_folder' )}", + url: "${h.url_for( controller='repository', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -61,7 +61,7 @@ type: "POST", url: "${h.url_for( controller='repository', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function ( data ) { cell.html( '' ) } From fcd8f50448cd1db7b50a58321c8fa391506bf997 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 20/35] Remove sample tracking manual external service transfer due to security concerns --- lib/galaxy/model/__init__.py | 25 -- .../galaxy/controllers/requests_admin.py | 296 +----------------- .../requests/select_datasets_to_transfer.mako | 149 --------- .../galaxy/requests/common/common.mako | 12 +- .../requests/common/view_request_history.mako | 1 - .../requests/common/view_sample_datasets.mako | 4 - 6 files changed, 4 insertions(+), 483 deletions(-) delete mode 100644 templates/admin/requests/select_datasets_to_transfer.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 23d92ead110..03e74cf3c07 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,15 +5,11 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -from galaxy import eggs -eggs.require("pexpect") - import codecs import errno import logging import operator import os -import pexpect import json import socket import time @@ -3937,27 +3933,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password']+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 10665404401..3f75afd54b6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -6,10 +6,7 @@ from galaxy.model.orm import * from galaxy import model, util from galaxy.web.form_builder import * from .requests_common import RequestsGrid, invalid_id_redirect -from galaxy import eggs -eggs.require("amqp") -import amqp -import logging, os, pexpect, ConfigParser +import logging, os, ConfigParser log = logging.getLogger( __name__ ) @@ -351,182 +348,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - files = [] - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
    ' ) ) - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names def __rename_dataset( self, sample, filepath, scp_configs ): name = filepath.split( '/' )[-1] options = sample.request.type.rename_dataset_options @@ -561,90 +382,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): flush_needed = True if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if not dataset_elements.has_key( external_service ): - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) \ - or not scp_configs.get( 'user_name', '' ) \ - or not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -686,35 +423,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
    %s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index 10d2135b7ef..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,149 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

    - - -%if not sample: -
    - Select a sample before selecting datasets to transfer -
    -%endif - -%if request.samples_without_library_destinations: -

    - Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

    -%endif - -%if message: - ${render_msg( message, status )} -%endif - -
    -
    Select datasets to transfer from data directory configured for the external service
    -
    -
    - - ${sample_id_select_field.get_html()} -
    - Select the sample that was sequenced to produce the datasets you want to transfer. -
    -
    -
    - -
    - Loading... -
    - -
    -
      -
    • Click the external service configuration button and change the Data directory setting to redefine the source data location.
      • -
    • Select a folder to select all of the individual files within that folder.
      • -
    • Click the Select datasets button when desired dataset check boxes are checked.
      • -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    - -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

    - ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

    - %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request
    • From 581808aa98a6058a4bdbe13c21271ca4b0d3ef77 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 21/35] Security fixes for tool shed hg push and capsule/tarball uploads --- .../tool_shed/framework/middleware/hg.py | 21 ++++++- lib/tool_shed/capsule/capsule_manager.py | 57 ++++++++++++------- lib/tool_shed/util/commit_util.py | 54 +++++++++++------- lib/tool_shed/util/repository_content_util.py | 26 +++------ 4 files changed, 99 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 14745cb6446..f07f017837c 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -9,7 +9,7 @@ from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import AUTH_TYPE from paste.httpheaders import REMOTE_USER -from galaxy.util import asbool +from galaxy.util import asbool, safe_relpath from galaxy.util.hash_util import new_secure_hash from tool_shed.util import hg_util import tool_shed.repository_types.util as rt_util @@ -113,7 +113,11 @@ class Hg( object ): fh.write( chunk ) fh.close() fh = open( tmp_filename, 'rb' ) - changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + try: + changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + except AttributeError: + msg = 'Your version of Mercurial is not supported. Please use a version < 3.5' + return self.__display_exception_remotely( start_response, msg ) fh.close() try: os.unlink( tmp_filename ) @@ -122,6 +126,19 @@ class Hg( object ): if changeset_groups: # Check the repository type to make sure inappropriate files are not being pushed. if 'PATH_INFO' in environ: + # Ensure there are no symlinks with targets outside the repo + for entry in changeset_groups: + if len( entry ) == 2: + filename, change_list = entry + if not isinstance(change_list, list): + change_list = [change_list] + for change in change_list: + for patch in change['data']: + target = patch['block'].strip() + if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ): + msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target ) + log.warning( msg ) + return self.__display_exception_remotely( start_response, msg ) # Instantiate a database connection engine = sqlalchemy.create_engine( self.db_url ) connection = engine.connect() diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 15b6fb43d87..642b04f33d9 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -12,6 +12,7 @@ from galaxy import web from galaxy.model.orm import and_ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE +from galaxy.util import safe_relpath from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder @@ -727,29 +728,16 @@ class ImportRepositoryManager( object ): repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, error_message = commit_util.check_archive( repository, archive ) - if ok: + check_results = commit_util.check_archive( repository, archive ) + # We filter out undesirable files but fail on undesriable dirs. Not + # sure why, just trying to maintain the same behavior as before. -nate + if not check_results.invalid and not check_results.undesirable_dirs: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in archive.getmembers(): - # Check files and directories in the archive. - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - error_message = 'Import failed: invalid file path %s in archive %s' % \ - ( str( file_path_item ), str( archive_file_name ) ) - results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message - return results_dict - filenames_in_archive.append( tarinfo_obj.name ) - else: - undesirable_files_removed += 1 # Extract the uploaded archive to the repository root. - archive.extractall( path=full_path ) + archive.extractall( path=full_path, members=check_results.valid ) archive.close() - for filename in filenames_in_archive: + for tar_member in check_results.valid: + filename = tar_member.name uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if toolshed or changeset_revision attributes @@ -776,6 +764,9 @@ class ImportRepositoryManager( object ): new_repo_alert = True # Since the repository is new, the following must be False. remove_repo_files_not_in_tar = False + filenames_in_archive = [ member.name for member in check_results.valid ] + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = 0 ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ commit_util.handle_directory_changes( self.app, self.host, @@ -806,7 +797,13 @@ class ImportRepositoryManager( object ): else: archive.close() results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message + results_dict[ 'error_message' ] += 'Capsule errors were found: ' + if check_results.invalid: + results_dict[ 'error_message' ] += '%s Invalid files were: %s.' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + if check_results.undesirable_dirs: + results_dict[ 'error_message' ] += ' Undesirable directories were: %s.' % ( + ', '.join( check_results.undesirable_dirs ) ) return results_dict def upload_capsule( self, **kwd ): @@ -863,6 +860,12 @@ class ImportRepositoryManager( object ): return_dict[ 'status' ] = 'error' uploaded_file.close() return return_dict + if not self.validate_archive_paths( tar_archive ): + return_dict[ 'status' ] = 'error' + return_dict[ 'message' ] = ( 'This capsule contains an invalid member type ' + 'or a file outside the archive path.' ) + uploaded_file.close() + return return_dict return_dict[ 'tar_archive' ] = tar_archive return_dict[ 'capsule_file_name' ] = uploaded_file_filename uploaded_file.close() @@ -872,6 +875,18 @@ class ImportRepositoryManager( object ): return return_dict return return_dict + def validate_archive_paths( self, tar_archive ): + ''' + Inspect the archive contents to ensure that there are no risky symlinks. + Returns True if a suspicious path is found. + ''' + for member in tar_archive.getmembers(): + if not ( member.isdir() or member.isfile() or member.islnk() ): + return False + elif not safe_relpath( member.name ): + return False + return True + def validate_capsule( self, **kwd ): """ Inspect the uploaded capsule's manifest and its contained files to ensure it is a valid diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index e6ff14ad1ec..a6eae096644 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -4,8 +4,10 @@ import logging import os import shutil import tempfile +from collections import namedtuple from galaxy.datatypes import checkers +from galaxy.util import safe_relpath from tool_shed.tools import data_table_manager @@ -21,30 +23,44 @@ UNDESIRABLE_DIRS = [ '.hg', '.svn', '.git', '.cvs' ] UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store', 'tool_test_output.html', 'tool_test_output.json' ] def check_archive( repository, archive ): + valid = [] + invalid = [] + errors = [] + undesirable_files = [] + undesirable_dirs = [] for member in archive.getmembers(): # Allow regular files and directories only if not ( member.isdir() or member.isfile() or member.islnk() ): - message = "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc). " - message += "The problematic member in this archive is %s," % str( member.name ) - return False, message - for item in [ '.hg', '..', '/' ]: - if member.name.startswith( item ): - message = "Uploaded archives cannot contain .hg directories, absolute filenames starting with '/', or filenames with two dots '..'. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message - if member.name in [ 'hgrc' ]: - message = "Uploaded archives cannot contain hgrc files. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message + errors.append( "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc)." ) + invalid.append( member ) + continue + if not safe_relpath( member.name ): + errors.append( "Uploaded archives cannot contain files that would extract outside of the archive." ) + invalid.append( member ) + continue + if os.path.basename( member.name ) in UNDESIRABLE_FILES: + undesirable_files.append( member ) + continue + head = tail = member.name + try: + while tail: + head, tail = os.path.split(head) + if tail in UNDESIRABLE_DIRS: + undesirable_dirs.append( member ) + assert False + except AssertionError: + continue if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message + errors.append( 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' ) + invalid.append( member ) + continue if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message - return True, '' + errors.append( 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' ) + invalid.append( member ) + continue + valid.append( member ) + ArchiveCheckResults = namedtuple( 'ArchiveCheckResults', [ 'valid', 'invalid', 'undesirable_files', 'undesirable_dirs', 'errors' ] ) + return ArchiveCheckResults( valid, invalid, undesirable_files, undesirable_dirs, errors ) def check_file_contents_for_email_alerts( app ): """ diff --git a/lib/tool_shed/util/repository_content_util.py b/lib/tool_shed/util/repository_content_util.py index 99fc9793b83..1cf6660278c 100644 --- a/lib/tool_shed/util/repository_content_util.py +++ b/lib/tool_shed/util/repository_content_util.py @@ -15,31 +15,23 @@ def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, message = commit_util.check_archive( repository, tar ) - if not ok: + check_results = commit_util.check_archive( repository, tar ) + if check_results.invalid: tar.close() uploaded_file.close() - return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed + message = '%s Invalid paths were: %s' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) else: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in tar.getmembers(): - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - ok = False - break - else: - undesirable_files_removed += 1 - if ok: - filenames_in_archive.append( tarinfo_obj.name ) + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = len( check_results.undesirable_dirs ) + filenames_in_archive = [ ti.name for ti in check_results.valid ] # Extract the uploaded tar to the load_point within the repository hierarchy. - tar.extractall( path=full_path ) + tar.extractall( path=full_path, members=check_results.valid ) tar.close() uploaded_file.close() for filename in filenames_in_archive: From 91f5fa81fe3c1c4d45b3987b27c187c4cb53f8e8 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 22/35] Security fixes for tool shed repository browsing --- .../galaxy/controllers/admin_toolshed.py | 8 ++-- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/util/shed_util_common.py | 46 +++++++++++++++---- .../admin/tool_shed_repository/common.mako | 12 +++-- .../webapps/tool_shed/repository/common.mako | 10 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index ee57e51e091..49902e78ccb 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -382,11 +382,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose @web.require_admin @@ -910,11 +910,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index b7f7bf71056..4a9ce32bcf4 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1635,11 +1635,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose def get_functional_test_rss( self, trans, **kwd ): @@ -2589,11 +2589,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 62751834765..08e2458a07b 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -535,9 +535,17 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha return repository -def get_repository_file_contents( file_path ): +def get_repository_file_contents( app, file_path, repository_id ): """Return the display-safe contents of a repository file for display in a browser.""" - if checkers.is_gzip( file_path ): + safe_str = '' + if not is_path_within_repo( app, file_path, repository_id ): + log.warning( 'Request tries to access a file outside of the repository location. File path: %s', file_path ) + return 'Invalid file path' + # Symlink targets are checked by is_path_within_repo + if os.path.islink( file_path ): + safe_str = 'link to: ' + basic_util.to_html_string( os.readlink( file_path ) ) + return safe_str + elif checkers.is_gzip( file_path ): return '
    gzip compressed file
    ' elif checkers.is_bz2( file_path ): return '
    bz2 compressed file
    ' @@ -546,7 +554,6 @@ def get_repository_file_contents( file_path ): elif checkers.check_binary( file_path ): return '
    Binary file
    ' else: - safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. @@ -556,6 +563,7 @@ def get_repository_file_contents( file_path ): util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. @@ -577,9 +585,6 @@ def get_repository_files( folder_path ): # Skip .hg directories if item.startswith( '.hg' ): continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item contents.append( item ) if contents: contents.sort() @@ -1054,11 +1059,15 @@ def is_tool_shed_client( app ): return hasattr( app, "install_model" ) -def open_repository_files_folder( folder_path ): +def open_repository_files_folder( app, folder_path, repository_id ): """ Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy. """ + # Symlink targets are checked by is_path_within_repo + if not is_path_within_repo( app, folder_path, repository_id ): + log.warning( 'Request tries to access a folder outside of the repository location. Folder path: %s', folder_path ) + return [] try: files_list = get_repository_files( folder_path ) except OSError, e: @@ -1068,10 +1077,17 @@ def open_repository_files_folder( folder_path ): folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[ -1 ] == os.sep: - is_folder = True + full_path = os.path.join( folder_path, filename ) + is_link = os.path.islink( full_path ) + path_is_within_repo = is_path_within_repo( app, full_path, repository_id ) + if is_link and not path_is_within_repo: + log.warning( 'Valid folder contains a symlink outside of the repository location. Link found in: ' + str( full_path ) ) if filename: - full_path = os.path.join( folder_path, filename ) + if os.path.isdir( full_path ) and path_is_within_repo: + # Append a '/' character so that our jquery dynatree will function properly. + filename = '%s/' % filename + full_path = '%s/' % full_path + is_folder = True node = { "title": filename, "isFolder": is_folder, "isLazy": is_folder, @@ -1081,6 +1097,16 @@ def open_repository_files_folder( folder_path ): return folder_contents +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folde ron the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path + + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 6cf501e1baa..ec06a65a6c3 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -23,14 +23,16 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${directory_path|h}" }, + dataType: "json", + data: { folder_path: "${directory_path|h}", + repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", + url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, + repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -55,7 +57,7 @@ type: "POST", url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function( data ) { cell.html( '' ) } diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index b6382de8047..ff3c15a85e2 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -22,14 +22,14 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='repository', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${repository.repo_path( trans.app )}" }, + dataType: "json", + data: { folder_path: "${repository.repo_path( trans.app )}", repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='repository', action='open_folder' )}", + url: "${h.url_for( controller='repository', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -62,7 +62,7 @@ type: "POST", url: "${h.url_for( controller='repository', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function ( data ) { cell.html( '' ) } From 2bda436473b116c7d98c36a8435056745bf832f2 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:20 -0500 Subject: [PATCH 23/35] Security fixes for tool shed hg push and capsule/tarball uploads --- .../tool_shed/framework/middleware/hg.py | 21 ++++++- lib/tool_shed/capsule/capsule_manager.py | 57 ++++++++++++------- lib/tool_shed/util/commit_util.py | 54 +++++++++++------- lib/tool_shed/util/repository_content_util.py | 26 +++------ 4 files changed, 99 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 6e2bfad7d73..921d43ca291 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -9,7 +9,7 @@ from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import AUTH_TYPE from paste.httpheaders import REMOTE_USER -from galaxy.util import asbool +from galaxy.util import asbool, safe_relpath from galaxy.util.hash_util import new_secure_hash from tool_shed.util import hg_util from tool_shed.util import commit_util @@ -114,7 +114,11 @@ class Hg( object ): fh.write( chunk ) fh.close() fh = open( tmp_filename, 'rb' ) - changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + try: + changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + except AttributeError: + msg = 'Your version of Mercurial is not supported. Please use a version < 3.5' + return self.__display_exception_remotely( start_response, msg ) fh.close() try: os.unlink( tmp_filename ) @@ -123,6 +127,19 @@ class Hg( object ): if changeset_groups: # Check the repository type to make sure inappropriate files are not being pushed. if 'PATH_INFO' in environ: + # Ensure there are no symlinks with targets outside the repo + for entry in changeset_groups: + if len( entry ) == 2: + filename, change_list = entry + if not isinstance(change_list, list): + change_list = [change_list] + for change in change_list: + for patch in change['data']: + target = patch['block'].strip() + if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ): + msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target ) + log.warning( msg ) + return self.__display_exception_remotely( start_response, msg ) # Instantiate a database connection engine = sqlalchemy.create_engine( self.db_url ) connection = engine.connect() diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 15b6fb43d87..642b04f33d9 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -12,6 +12,7 @@ from galaxy import web from galaxy.model.orm import and_ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE +from galaxy.util import safe_relpath from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder @@ -727,29 +728,16 @@ class ImportRepositoryManager( object ): repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, error_message = commit_util.check_archive( repository, archive ) - if ok: + check_results = commit_util.check_archive( repository, archive ) + # We filter out undesirable files but fail on undesriable dirs. Not + # sure why, just trying to maintain the same behavior as before. -nate + if not check_results.invalid and not check_results.undesirable_dirs: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in archive.getmembers(): - # Check files and directories in the archive. - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - error_message = 'Import failed: invalid file path %s in archive %s' % \ - ( str( file_path_item ), str( archive_file_name ) ) - results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message - return results_dict - filenames_in_archive.append( tarinfo_obj.name ) - else: - undesirable_files_removed += 1 # Extract the uploaded archive to the repository root. - archive.extractall( path=full_path ) + archive.extractall( path=full_path, members=check_results.valid ) archive.close() - for filename in filenames_in_archive: + for tar_member in check_results.valid: + filename = tar_member.name uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if toolshed or changeset_revision attributes @@ -776,6 +764,9 @@ class ImportRepositoryManager( object ): new_repo_alert = True # Since the repository is new, the following must be False. remove_repo_files_not_in_tar = False + filenames_in_archive = [ member.name for member in check_results.valid ] + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = 0 ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ commit_util.handle_directory_changes( self.app, self.host, @@ -806,7 +797,13 @@ class ImportRepositoryManager( object ): else: archive.close() results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message + results_dict[ 'error_message' ] += 'Capsule errors were found: ' + if check_results.invalid: + results_dict[ 'error_message' ] += '%s Invalid files were: %s.' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + if check_results.undesirable_dirs: + results_dict[ 'error_message' ] += ' Undesirable directories were: %s.' % ( + ', '.join( check_results.undesirable_dirs ) ) return results_dict def upload_capsule( self, **kwd ): @@ -863,6 +860,12 @@ class ImportRepositoryManager( object ): return_dict[ 'status' ] = 'error' uploaded_file.close() return return_dict + if not self.validate_archive_paths( tar_archive ): + return_dict[ 'status' ] = 'error' + return_dict[ 'message' ] = ( 'This capsule contains an invalid member type ' + 'or a file outside the archive path.' ) + uploaded_file.close() + return return_dict return_dict[ 'tar_archive' ] = tar_archive return_dict[ 'capsule_file_name' ] = uploaded_file_filename uploaded_file.close() @@ -872,6 +875,18 @@ class ImportRepositoryManager( object ): return return_dict return return_dict + def validate_archive_paths( self, tar_archive ): + ''' + Inspect the archive contents to ensure that there are no risky symlinks. + Returns True if a suspicious path is found. + ''' + for member in tar_archive.getmembers(): + if not ( member.isdir() or member.isfile() or member.islnk() ): + return False + elif not safe_relpath( member.name ): + return False + return True + def validate_capsule( self, **kwd ): """ Inspect the uploaded capsule's manifest and its contained files to ensure it is a valid diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index f0cca42f132..ce81870517c 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -5,8 +5,10 @@ import logging import os import shutil import tempfile +from collections import namedtuple from galaxy.datatypes import checkers +from galaxy.util import safe_relpath import tool_shed.repository_types.util as rt_util from tool_shed.tools import data_table_manager @@ -21,30 +23,44 @@ UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store', 'tool_test_output def check_archive( repository, archive ): + valid = [] + invalid = [] + errors = [] + undesirable_files = [] + undesirable_dirs = [] for member in archive.getmembers(): # Allow regular files and directories only if not ( member.isdir() or member.isfile() or member.islnk() ): - message = "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc). " - message += "The problematic member in this archive is %s," % str( member.name ) - return False, message - for item in [ '.hg', '..', '/' ]: - if member.name.startswith( item ): - message = "Uploaded archives cannot contain .hg directories, absolute filenames starting with '/', or filenames with two dots '..'. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message - if member.name in [ 'hgrc' ]: - message = "Uploaded archives cannot contain hgrc files. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message + errors.append( "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc)." ) + invalid.append( member ) + continue + if not safe_relpath( member.name ): + errors.append( "Uploaded archives cannot contain files that would extract outside of the archive." ) + invalid.append( member ) + continue + if os.path.basename( member.name ) in UNDESIRABLE_FILES: + undesirable_files.append( member ) + continue + head = tail = member.name + try: + while tail: + head, tail = os.path.split(head) + if tail in UNDESIRABLE_DIRS: + undesirable_dirs.append( member ) + assert False + except AssertionError: + continue if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message + errors.append( 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' ) + invalid.append( member ) + continue if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message - return True, '' + errors.append( 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' ) + invalid.append( member ) + continue + valid.append( member ) + ArchiveCheckResults = namedtuple( 'ArchiveCheckResults', [ 'valid', 'invalid', 'undesirable_files', 'undesirable_dirs', 'errors' ] ) + return ArchiveCheckResults( valid, invalid, undesirable_files, undesirable_dirs, errors ) def check_file_contents_for_email_alerts( app ): diff --git a/lib/tool_shed/util/repository_content_util.py b/lib/tool_shed/util/repository_content_util.py index 99fc9793b83..1cf6660278c 100644 --- a/lib/tool_shed/util/repository_content_util.py +++ b/lib/tool_shed/util/repository_content_util.py @@ -15,31 +15,23 @@ def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, message = commit_util.check_archive( repository, tar ) - if not ok: + check_results = commit_util.check_archive( repository, tar ) + if check_results.invalid: tar.close() uploaded_file.close() - return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed + message = '%s Invalid paths were: %s' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) else: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in tar.getmembers(): - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - ok = False - break - else: - undesirable_files_removed += 1 - if ok: - filenames_in_archive.append( tarinfo_obj.name ) + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = len( check_results.undesirable_dirs ) + filenames_in_archive = [ ti.name for ti in check_results.valid ] # Extract the uploaded tar to the load_point within the repository hierarchy. - tar.extractall( path=full_path ) + tar.extractall( path=full_path, members=check_results.valid ) tar.close() uploaded_file.close() for filename in filenames_in_archive: From 47a8b6e86733472ad3c0dbe2d5774d69ecba44eb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:21 -0500 Subject: [PATCH 24/35] Add a safe_relpath util function for ensuring a path does not reference an absolute or parent directory --- lib/galaxy/util/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 39f84532aee..f17362769ca 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -29,7 +29,7 @@ from datetime import datetime from email.MIMEText import MIMEText -from os.path import relpath +from os.path import relpath, normpath from hashlib import md5 from itertools import izip @@ -1252,6 +1252,22 @@ def galaxy_directory(): return os.path.abspath(galaxy_root_path) +def safe_relpath(path): + """ + Given what we expect to be a relative path, determine whether the path + would exist inside the current directory. + + :type path: string + :param path: a path to check + :rtype: bool + :returns: ``True`` if path is relative and does not reference a path + in a parent directory, ``False`` otherwise. + """ + if path.startswith(os.sep) or normpath(path).startswith(os.pardir): + return False + return True + + class ExecutionTimer(object): def __init__(self): From 6190f6fa6db26ada3380c9a54b5481d12b3ac9ae Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:21 -0500 Subject: [PATCH 25/35] Security fixes for history imports --- lib/galaxy/exceptions/__init__.py | 5 ++ lib/galaxy/exceptions/error_codes.json | 7 ++- lib/galaxy/tools/imp_exp/__init__.py | 6 ++- .../tools/imp_exp/unpack_tar_gz_archive.py | 53 +++++++++++++------ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 65210229d55..4421d994e76 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -71,6 +71,11 @@ class MalformedId( MessageException ): err_code = error_codes.MALFORMED_ID +class MalformedContents( MessageException ): + status_code = 400 + err_code = error_codes.MALFORMED_CONTENTS + + class UnknownContentsType( MessageException ): status_code = 400 err_code = error_codes.UNKNOWN_CONTENTS_TYPE diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index c681afdebb0..50faf0fc644 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -61,9 +61,14 @@ }, { "name": "USER_TOOL_META_PARAMETER_PROBLEM", - "code": 400011, + "code": 400012, "message": "Supplied incorrect or incompatible tool meta parameters." }, + { + "name": "MALFORMED_CONTENTS", + "code": 400013, + "message": "The contents of the request are malformed." + }, { "name": "USER_AUTHENTICATION_FAILED", "code": 401001, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 364e9351179..f5a5c7ddb2b 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -5,6 +5,7 @@ import tempfile import json import datetime from galaxy import model +from galaxy.exceptions import MalformedContents from galaxy.model.item_attrs import UsesAnnotations from galaxy.model.orm import eagerload, eagerload_all from galaxy.tools.parameters.basic import UnvalidatedValue @@ -192,9 +193,9 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): if dataset_attrs.get('exported', True) is True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ - os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) + os.path.realpath( os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) ) if not file_in_dir( temp_dataset_file_name, os.path.join( archive_dir, "datasets" ) ): - raise Exception( "Invalid dataset path: %s" % temp_dataset_file_name ) + raise MalformedContents( "Invalid dataset path: %s" % temp_dataset_file_name ) if datasets_usage_counts[ temp_dataset_file_name ] == 1: shutil.move( temp_dataset_file_name, hda.file_name ) else: @@ -316,6 +317,7 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + raise class JobExportHistoryArchiveWrapper( object, UsesAnnotations ): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index d6890257c8c..f3dab0c5ae4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,6 +6,7 @@ usage: %prog archive_source dest_dir --[url|file] source type, either a URL or a file. """ +import os import sys import optparse import tarfile @@ -43,6 +44,22 @@ def url_to_file( url, dest_file ): return None +def check_archive( archive_file, dest_dir ): + """ + Ensure that a tar archive has no absolute paths or relative paths outside + the archive. + """ + with tarfile.open( archive_file, mode='r:gz' ) as archive_fp: + for arc_path in archive_fp.getnames(): + assert os.path.normpath( + os.path.join( + dest_dir, + arc_path + ) ).startswith( dest_dir.rstrip(os.sep) + os.sep ), \ + "Archive member would extract outside target directory: %s" % arc_path + return True + + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -51,13 +68,8 @@ def unpack_archive( archive_file, dest_dir ): archive_fp.extractall( path=dest_dir ) archive_fp.close() -if __name__ == "__main__": - # Parse command line. - parser = optparse.OptionParser() - parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) - parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) - (options, args) = parser.parse_args() + +def main(options, args): is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args @@ -66,14 +78,25 @@ if __name__ == "__main__": archive_source = b64decode( archive_source ) dest_dir = b64decode( dest_dir ) - try: - # Get archive from URL. - if is_url: - archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) - elif is_file: - archive_file = archive_source + # Get archive from URL. + if is_url: + archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) + elif is_file: + archive_file = archive_source - # Unpack archive. - unpack_archive( archive_file, dest_dir ) + # Unpack archive. + check_archive( archive_file, dest_dir ) + unpack_archive( archive_file, dest_dir ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) + parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) + parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) + (options, args) = parser.parse_args() + try: + main(options, args) except Exception, e: print "Error unpacking tar/gz archive: %s" % e, sys.stderr From 27d7298ec4788a2e59a5988007fde05e5fc004ce Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:21 -0500 Subject: [PATCH 26/35] Security fixes for object store paths --- lib/galaxy/objectstore/__init__.py | 21 ++++++++++++++++----- lib/galaxy/objectstore/rods.py | 17 ++++++++++++++++- lib/galaxy/objectstore/s3.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index b1a4fc33546..1438e577bbc 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,7 +11,7 @@ import logging import threading from xml.etree import ElementTree -from galaxy.util import umask_fix_perms, force_symlink +from galaxy.util import umask_fix_perms, force_symlink, safe_relpath from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util.sleeper import Sleeper from galaxy.util.directory_hash import directory_hash_id @@ -252,7 +252,17 @@ class DiskObjectStore(ObjectStore): the composed directory structure does not include a hash id (e.g., /files/dataset_10.dat (old) vs. /files/000/dataset_10.dat (new)) """ - base = self.extra_dirs.get(base_dir, self.file_path) + base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path)) + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name and not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") if old_style: if extra_dir is not None: path = os.path.join(base, extra_dir) @@ -619,9 +629,10 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 'irods': from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'pulsar': - from .pulsar import PulsarObjectStore - return PulsarObjectStore(config=config, config_xml=config_xml) + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4754156ad5b..f4bb03c2706 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,7 +12,8 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.exceptions import ObjectNotFound +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import safe_relpath from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs try: @@ -71,6 +72,20 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but iRODS will + # not follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) path = "" if extra_dir is not None: path = extra_dir diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index a5215ec1963..169867541e4 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -12,8 +12,8 @@ import time from datetime import datetime -from galaxy.exceptions import ObjectNotFound -from galaxy.util import string_as_bool, umask_fix_perms +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import string_as_bool, umask_fix_perms, safe_relpath from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sleeper import Sleeper from .s3_multipart_upload import multipart_upload @@ -208,6 +208,20 @@ class S3ObjectStore(ObjectStore): umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, base_dir=None, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) rel_path = os.path.join(*directory_hash_id(obj.id)) if extra_dir is not None: if extra_dir_at_root: From c465489ea1676905ddb4cfb88605c1e869a7019c Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:21 -0500 Subject: [PATCH 27/35] Remove sample tracking manual external service transfer due to security concerns --- lib/galaxy/model/__init__.py | 25 -- .../galaxy/controllers/requests_admin.py | 295 +----------------- .../requests/select_datasets_to_transfer.mako | 149 --------- .../galaxy/requests/common/common.mako | 12 +- .../requests/common/view_request_history.mako | 1 - .../requests/common/view_sample_datasets.mako | 4 - 6 files changed, 4 insertions(+), 482 deletions(-) delete mode 100644 templates/admin/requests/select_datasets_to_transfer.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2bd9788899f..f50314db20a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,15 +5,11 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -from galaxy import eggs -eggs.require("pexpect") - import codecs import errno import logging import operator import os -import pexpect import json import socket import time @@ -3975,27 +3971,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password']+'\r\n', - pexpect.TIMEOUT:print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 489143b5980..f7990c94f3a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -9,9 +9,7 @@ from .requests_common import RequestsGrid, invalid_id_redirect from galaxy import eggs eggs.require( "MarkupSafe" ) from markupsafe import escape -eggs.require("amqp") -import amqp -import logging, os, pexpect, ConfigParser +import logging, os, ConfigParser log = logging.getLogger( __name__ ) @@ -353,182 +351,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - files = [] - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
    ' ) ) - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names def __rename_dataset( self, sample, filepath, scp_configs ): name = filepath.split( '/' )[-1] options = sample.request.type.rename_dataset_options @@ -563,90 +385,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): flush_needed = True if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if not dataset_elements.has_key( external_service ): - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) \ - or not scp_configs.get( 'user_name', '' ) \ - or not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -688,35 +426,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
    %s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index 10d2135b7ef..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,149 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

    - - -%if not sample: -
    - Select a sample before selecting datasets to transfer -
    -%endif - -%if request.samples_without_library_destinations: -

    - Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

    -%endif - -%if message: - ${render_msg( message, status )} -%endif - -
    -
    Select datasets to transfer from data directory configured for the external service
    -
    -
    - - ${sample_id_select_field.get_html()} -
    - Select the sample that was sequenced to produce the datasets you want to transfer. -
    -
    -
    - -
    - Loading... -
    - -
    -
      -
    • Click the external service configuration button and change the Data directory setting to redefine the source data location.
      • -
    • Select a folder to select all of the individual files within that folder.
      • -
    • Click the Select datasets button when desired dataset check boxes are checked.
      • -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    - -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

    - ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

    - %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request
    • From 18b3126a15ec7fe3873253be0c1344e9f067f0ca Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:21 -0500 Subject: [PATCH 28/35] Security fixes for tool shed repository browsing --- .../galaxy/controllers/admin_toolshed.py | 8 ++-- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/util/shed_util_common.py | 46 +++++++++++++++---- .../admin/tool_shed_repository/common.mako | 12 +++-- .../webapps/tool_shed/repository/common.mako | 10 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index ee57e51e091..49902e78ccb 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -382,11 +382,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose @web.require_admin @@ -910,11 +910,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 188e35329b6..15206dc008b 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1636,11 +1636,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose def get_functional_test_rss( self, trans, **kwd ): @@ -2590,11 +2590,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 62751834765..08e2458a07b 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -535,9 +535,17 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha return repository -def get_repository_file_contents( file_path ): +def get_repository_file_contents( app, file_path, repository_id ): """Return the display-safe contents of a repository file for display in a browser.""" - if checkers.is_gzip( file_path ): + safe_str = '' + if not is_path_within_repo( app, file_path, repository_id ): + log.warning( 'Request tries to access a file outside of the repository location. File path: %s', file_path ) + return 'Invalid file path' + # Symlink targets are checked by is_path_within_repo + if os.path.islink( file_path ): + safe_str = 'link to: ' + basic_util.to_html_string( os.readlink( file_path ) ) + return safe_str + elif checkers.is_gzip( file_path ): return '
    gzip compressed file
    ' elif checkers.is_bz2( file_path ): return '
    bz2 compressed file
    ' @@ -546,7 +554,6 @@ def get_repository_file_contents( file_path ): elif checkers.check_binary( file_path ): return '
    Binary file
    ' else: - safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. @@ -556,6 +563,7 @@ def get_repository_file_contents( file_path ): util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. @@ -577,9 +585,6 @@ def get_repository_files( folder_path ): # Skip .hg directories if item.startswith( '.hg' ): continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item contents.append( item ) if contents: contents.sort() @@ -1054,11 +1059,15 @@ def is_tool_shed_client( app ): return hasattr( app, "install_model" ) -def open_repository_files_folder( folder_path ): +def open_repository_files_folder( app, folder_path, repository_id ): """ Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy. """ + # Symlink targets are checked by is_path_within_repo + if not is_path_within_repo( app, folder_path, repository_id ): + log.warning( 'Request tries to access a folder outside of the repository location. Folder path: %s', folder_path ) + return [] try: files_list = get_repository_files( folder_path ) except OSError, e: @@ -1068,10 +1077,17 @@ def open_repository_files_folder( folder_path ): folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[ -1 ] == os.sep: - is_folder = True + full_path = os.path.join( folder_path, filename ) + is_link = os.path.islink( full_path ) + path_is_within_repo = is_path_within_repo( app, full_path, repository_id ) + if is_link and not path_is_within_repo: + log.warning( 'Valid folder contains a symlink outside of the repository location. Link found in: ' + str( full_path ) ) if filename: - full_path = os.path.join( folder_path, filename ) + if os.path.isdir( full_path ) and path_is_within_repo: + # Append a '/' character so that our jquery dynatree will function properly. + filename = '%s/' % filename + full_path = '%s/' % full_path + is_folder = True node = { "title": filename, "isFolder": is_folder, "isLazy": is_folder, @@ -1081,6 +1097,16 @@ def open_repository_files_folder( folder_path ): return folder_contents +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folde ron the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path + + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 6cf501e1baa..ec06a65a6c3 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -23,14 +23,16 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${directory_path|h}" }, + dataType: "json", + data: { folder_path: "${directory_path|h}", + repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", + url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, + repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -55,7 +57,7 @@ type: "POST", url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function( data ) { cell.html( '' ) } diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index cd892fdc74a..4e67c86127d 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -22,14 +22,14 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='repository', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${repository.repo_path( trans.app )}" }, + dataType: "json", + data: { folder_path: "${repository.repo_path( trans.app )}", repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='repository', action='open_folder' )}", + url: "${h.url_for( controller='repository', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -62,7 +62,7 @@ type: "POST", url: "${h.url_for( controller='repository', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function ( data ) { cell.html( '' ) } From 40a90831ed28b762e90cf2b50abec7e8bef738d6 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:21 -0500 Subject: [PATCH 29/35] Security fixes for tool shed hg push and capsule/tarball uploads --- .../tool_shed/framework/middleware/hg.py | 21 ++++++- lib/tool_shed/capsule/capsule_manager.py | 57 ++++++++++++------- lib/tool_shed/util/commit_util.py | 54 +++++++++++------- lib/tool_shed/util/repository_content_util.py | 26 +++------ 4 files changed, 99 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 6e2bfad7d73..921d43ca291 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -9,7 +9,7 @@ from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import AUTH_TYPE from paste.httpheaders import REMOTE_USER -from galaxy.util import asbool +from galaxy.util import asbool, safe_relpath from galaxy.util.hash_util import new_secure_hash from tool_shed.util import hg_util from tool_shed.util import commit_util @@ -114,7 +114,11 @@ class Hg( object ): fh.write( chunk ) fh.close() fh = open( tmp_filename, 'rb' ) - changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + try: + changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + except AttributeError: + msg = 'Your version of Mercurial is not supported. Please use a version < 3.5' + return self.__display_exception_remotely( start_response, msg ) fh.close() try: os.unlink( tmp_filename ) @@ -123,6 +127,19 @@ class Hg( object ): if changeset_groups: # Check the repository type to make sure inappropriate files are not being pushed. if 'PATH_INFO' in environ: + # Ensure there are no symlinks with targets outside the repo + for entry in changeset_groups: + if len( entry ) == 2: + filename, change_list = entry + if not isinstance(change_list, list): + change_list = [change_list] + for change in change_list: + for patch in change['data']: + target = patch['block'].strip() + if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ): + msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target ) + log.warning( msg ) + return self.__display_exception_remotely( start_response, msg ) # Instantiate a database connection engine = sqlalchemy.create_engine( self.db_url ) connection = engine.connect() diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 15b6fb43d87..642b04f33d9 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -12,6 +12,7 @@ from galaxy import web from galaxy.model.orm import and_ from galaxy.util import asbool from galaxy.util import CHUNK_SIZE +from galaxy.util import safe_relpath from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder @@ -727,29 +728,16 @@ class ImportRepositoryManager( object ): repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, error_message = commit_util.check_archive( repository, archive ) - if ok: + check_results = commit_util.check_archive( repository, archive ) + # We filter out undesirable files but fail on undesriable dirs. Not + # sure why, just trying to maintain the same behavior as before. -nate + if not check_results.invalid and not check_results.undesirable_dirs: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in archive.getmembers(): - # Check files and directories in the archive. - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - error_message = 'Import failed: invalid file path %s in archive %s' % \ - ( str( file_path_item ), str( archive_file_name ) ) - results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message - return results_dict - filenames_in_archive.append( tarinfo_obj.name ) - else: - undesirable_files_removed += 1 # Extract the uploaded archive to the repository root. - archive.extractall( path=full_path ) + archive.extractall( path=full_path, members=check_results.valid ) archive.close() - for filename in filenames_in_archive: + for tar_member in check_results.valid: + filename = tar_member.name uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if toolshed or changeset_revision attributes @@ -776,6 +764,9 @@ class ImportRepositoryManager( object ): new_repo_alert = True # Since the repository is new, the following must be False. remove_repo_files_not_in_tar = False + filenames_in_archive = [ member.name for member in check_results.valid ] + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = 0 ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ commit_util.handle_directory_changes( self.app, self.host, @@ -806,7 +797,13 @@ class ImportRepositoryManager( object ): else: archive.close() results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message + results_dict[ 'error_message' ] += 'Capsule errors were found: ' + if check_results.invalid: + results_dict[ 'error_message' ] += '%s Invalid files were: %s.' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + if check_results.undesirable_dirs: + results_dict[ 'error_message' ] += ' Undesirable directories were: %s.' % ( + ', '.join( check_results.undesirable_dirs ) ) return results_dict def upload_capsule( self, **kwd ): @@ -863,6 +860,12 @@ class ImportRepositoryManager( object ): return_dict[ 'status' ] = 'error' uploaded_file.close() return return_dict + if not self.validate_archive_paths( tar_archive ): + return_dict[ 'status' ] = 'error' + return_dict[ 'message' ] = ( 'This capsule contains an invalid member type ' + 'or a file outside the archive path.' ) + uploaded_file.close() + return return_dict return_dict[ 'tar_archive' ] = tar_archive return_dict[ 'capsule_file_name' ] = uploaded_file_filename uploaded_file.close() @@ -872,6 +875,18 @@ class ImportRepositoryManager( object ): return return_dict return return_dict + def validate_archive_paths( self, tar_archive ): + ''' + Inspect the archive contents to ensure that there are no risky symlinks. + Returns True if a suspicious path is found. + ''' + for member in tar_archive.getmembers(): + if not ( member.isdir() or member.isfile() or member.islnk() ): + return False + elif not safe_relpath( member.name ): + return False + return True + def validate_capsule( self, **kwd ): """ Inspect the uploaded capsule's manifest and its contained files to ensure it is a valid diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index f0cca42f132..ce81870517c 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -5,8 +5,10 @@ import logging import os import shutil import tempfile +from collections import namedtuple from galaxy.datatypes import checkers +from galaxy.util import safe_relpath import tool_shed.repository_types.util as rt_util from tool_shed.tools import data_table_manager @@ -21,30 +23,44 @@ UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store', 'tool_test_output def check_archive( repository, archive ): + valid = [] + invalid = [] + errors = [] + undesirable_files = [] + undesirable_dirs = [] for member in archive.getmembers(): # Allow regular files and directories only if not ( member.isdir() or member.isfile() or member.islnk() ): - message = "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc). " - message += "The problematic member in this archive is %s," % str( member.name ) - return False, message - for item in [ '.hg', '..', '/' ]: - if member.name.startswith( item ): - message = "Uploaded archives cannot contain .hg directories, absolute filenames starting with '/', or filenames with two dots '..'. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message - if member.name in [ 'hgrc' ]: - message = "Uploaded archives cannot contain hgrc files. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message + errors.append( "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc)." ) + invalid.append( member ) + continue + if not safe_relpath( member.name ): + errors.append( "Uploaded archives cannot contain files that would extract outside of the archive." ) + invalid.append( member ) + continue + if os.path.basename( member.name ) in UNDESIRABLE_FILES: + undesirable_files.append( member ) + continue + head = tail = member.name + try: + while tail: + head, tail = os.path.split(head) + if tail in UNDESIRABLE_DIRS: + undesirable_dirs.append( member ) + assert False + except AssertionError: + continue if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message + errors.append( 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' ) + invalid.append( member ) + continue if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message - return True, '' + errors.append( 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' ) + invalid.append( member ) + continue + valid.append( member ) + ArchiveCheckResults = namedtuple( 'ArchiveCheckResults', [ 'valid', 'invalid', 'undesirable_files', 'undesirable_dirs', 'errors' ] ) + return ArchiveCheckResults( valid, invalid, undesirable_files, undesirable_dirs, errors ) def check_file_contents_for_email_alerts( app ): diff --git a/lib/tool_shed/util/repository_content_util.py b/lib/tool_shed/util/repository_content_util.py index 99fc9793b83..1cf6660278c 100644 --- a/lib/tool_shed/util/repository_content_util.py +++ b/lib/tool_shed/util/repository_content_util.py @@ -15,31 +15,23 @@ def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, message = commit_util.check_archive( repository, tar ) - if not ok: + check_results = commit_util.check_archive( repository, tar ) + if check_results.invalid: tar.close() uploaded_file.close() - return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed + message = '%s Invalid paths were: %s' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) else: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in tar.getmembers(): - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - ok = False - break - else: - undesirable_files_removed += 1 - if ok: - filenames_in_archive.append( tarinfo_obj.name ) + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = len( check_results.undesirable_dirs ) + filenames_in_archive = [ ti.name for ti in check_results.valid ] # Extract the uploaded tar to the load_point within the repository hierarchy. - tar.extractall( path=full_path ) + tar.extractall( path=full_path, members=check_results.valid ) tar.close() uploaded_file.close() for filename in filenames_in_archive: From b3e0315e1e40912366c8103412e64a645e0a93a9 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:22 -0500 Subject: [PATCH 30/35] Add a safe_relpath util function for ensuring a path does not reference an absolute or parent directory --- lib/galaxy/util/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 960eb81a05d..f27dfd4cea9 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -29,7 +29,7 @@ from datetime import datetime from email.MIMEText import MIMEText -from os.path import relpath +from os.path import relpath, normpath from hashlib import md5 from itertools import izip @@ -1277,6 +1277,22 @@ def parse_int(value, min_val=None, max_val=None, default=None, allow_none=False) raise +def safe_relpath(path): + """ + Given what we expect to be a relative path, determine whether the path + would exist inside the current directory. + + :type path: string + :param path: a path to check + :rtype: bool + :returns: ``True`` if path is relative and does not reference a path + in a parent directory, ``False`` otherwise. + """ + if path.startswith(os.sep) or normpath(path).startswith(os.pardir): + return False + return True + + class ExecutionTimer(object): def __init__(self): From 8736c7bd77bd3c5cd38225538085c0eb2a40e8f4 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:22 -0500 Subject: [PATCH 31/35] Security fixes for history imports --- lib/galaxy/exceptions/__init__.py | 5 ++ lib/galaxy/exceptions/error_codes.json | 7 ++- lib/galaxy/tools/imp_exp/__init__.py | 6 ++- .../tools/imp_exp/unpack_tar_gz_archive.py | 53 +++++++++++++------ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index ea708600dba..0c270de3ee0 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -70,6 +70,11 @@ class MalformedId( MessageException ): err_code = error_codes.MALFORMED_ID +class MalformedContents( MessageException ): + status_code = 400 + err_code = error_codes.MALFORMED_CONTENTS + + class UnknownContentsType( MessageException ): status_code = 400 err_code = error_codes.UNKNOWN_CONTENTS_TYPE diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index c681afdebb0..50faf0fc644 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -61,9 +61,14 @@ }, { "name": "USER_TOOL_META_PARAMETER_PROBLEM", - "code": 400011, + "code": 400012, "message": "Supplied incorrect or incompatible tool meta parameters." }, + { + "name": "MALFORMED_CONTENTS", + "code": 400013, + "message": "The contents of the request are malformed." + }, { "name": "USER_AUTHENTICATION_FAILED", "code": 401001, diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index fea5325f39a..55349a4149b 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -10,6 +10,7 @@ eggs.require('SQLAlchemy') from sqlalchemy.orm import eagerload, eagerload_all from galaxy import model +from galaxy.exceptions import MalformedContents from galaxy.model.item_attrs import UsesAnnotations from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.util.json import dumps, loads @@ -170,9 +171,9 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): if dataset_attrs.get('exported', True) is True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ - os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) + os.path.realpath( os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) ) if not file_in_dir( temp_dataset_file_name, os.path.join( archive_dir, "datasets" ) ): - raise Exception( "Invalid dataset path: %s" % temp_dataset_file_name ) + raise MalformedContents( "Invalid dataset path: %s" % temp_dataset_file_name ) if datasets_usage_counts[ temp_dataset_file_name ] == 1: self.app.object_store.update_from_file( hda.dataset, file_name=temp_dataset_file_name, create=True ) @@ -309,6 +310,7 @@ class JobImportHistoryArchiveWrapper( object, UsesAnnotations ): except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() + raise class JobExportHistoryArchiveWrapper( object, UsesAnnotations ): diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index d6890257c8c..f3dab0c5ae4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -6,6 +6,7 @@ usage: %prog archive_source dest_dir --[url|file] source type, either a URL or a file. """ +import os import sys import optparse import tarfile @@ -43,6 +44,22 @@ def url_to_file( url, dest_file ): return None +def check_archive( archive_file, dest_dir ): + """ + Ensure that a tar archive has no absolute paths or relative paths outside + the archive. + """ + with tarfile.open( archive_file, mode='r:gz' ) as archive_fp: + for arc_path in archive_fp.getnames(): + assert os.path.normpath( + os.path.join( + dest_dir, + arc_path + ) ).startswith( dest_dir.rstrip(os.sep) + os.sep ), \ + "Archive member would extract outside target directory: %s" % arc_path + return True + + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -51,13 +68,8 @@ def unpack_archive( archive_file, dest_dir ): archive_fp.extractall( path=dest_dir ) archive_fp.close() -if __name__ == "__main__": - # Parse command line. - parser = optparse.OptionParser() - parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) - parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) - (options, args) = parser.parse_args() + +def main(options, args): is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args @@ -66,14 +78,25 @@ if __name__ == "__main__": archive_source = b64decode( archive_source ) dest_dir = b64decode( dest_dir ) - try: - # Get archive from URL. - if is_url: - archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) - elif is_file: - archive_file = archive_source + # Get archive from URL. + if is_url: + archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) + elif is_file: + archive_file = archive_source - # Unpack archive. - unpack_archive( archive_file, dest_dir ) + # Unpack archive. + check_archive( archive_file, dest_dir ) + unpack_archive( archive_file, dest_dir ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) + parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) + parser.add_option( '-e', '--encoded', dest='is_b64encoded', action="store_true", default=False, help='Source and destination dir values are base64 encoded.' ) + (options, args) = parser.parse_args() + try: + main(options, args) except Exception, e: print "Error unpacking tar/gz archive: %s" % e, sys.stderr From 23e203c48180b6ab1e4886be81447e33afb853e5 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:22 -0500 Subject: [PATCH 32/35] Security fixes for object store paths --- lib/galaxy/objectstore/__init__.py | 21 ++++++++++++++++----- lib/galaxy/objectstore/rods.py | 17 ++++++++++++++++- lib/galaxy/objectstore/s3.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index b1a4fc33546..1438e577bbc 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -11,7 +11,7 @@ import logging import threading from xml.etree import ElementTree -from galaxy.util import umask_fix_perms, force_symlink +from galaxy.util import umask_fix_perms, force_symlink, safe_relpath from galaxy.exceptions import ObjectInvalid, ObjectNotFound from galaxy.util.sleeper import Sleeper from galaxy.util.directory_hash import directory_hash_id @@ -252,7 +252,17 @@ class DiskObjectStore(ObjectStore): the composed directory structure does not include a hash id (e.g., /files/dataset_10.dat (old) vs. /files/000/dataset_10.dat (new)) """ - base = self.extra_dirs.get(base_dir, self.file_path) + base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path)) + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name and not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") if old_style: if extra_dir is not None: path = os.path.join(base, extra_dir) @@ -619,9 +629,10 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 'irods': from .rods import IRODSObjectStore return IRODSObjectStore(config=config, config_xml=config_xml) - elif store == 'pulsar': - from .pulsar import PulsarObjectStore - return PulsarObjectStore(config=config, config_xml=config_xml) + # Disable the Pulsar object store for now until it receives some attention + # elif store == 'pulsar': + # from .pulsar import PulsarObjectStore + # return PulsarObjectStore(config=config, config_xml=config_xml) else: log.error("Unrecognized object store definition: {0}".format(store)) diff --git a/lib/galaxy/objectstore/rods.py b/lib/galaxy/objectstore/rods.py index 4754156ad5b..f4bb03c2706 100644 --- a/lib/galaxy/objectstore/rods.py +++ b/lib/galaxy/objectstore/rods.py @@ -12,7 +12,8 @@ from posixpath import join as path_join from posixpath import basename as path_basename from posixpath import dirname as path_dirname -from galaxy.exceptions import ObjectNotFound +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import safe_relpath from ..objectstore import DiskObjectStore, ObjectStore, local_extra_dirs try: @@ -71,6 +72,20 @@ class IRODSObjectStore( DiskObjectStore, ObjectStore ): log.info( "iRODS data for this instance will be stored in collection: %s, resource: %s", self.root_collection_path, self.default_resource ) def __get_rods_path( self, obj, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, strip_dat=True, **kwargs ): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but iRODS will + # not follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) path = "" if extra_dir is not None: path = extra_dir diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index a5215ec1963..169867541e4 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -12,8 +12,8 @@ import time from datetime import datetime -from galaxy.exceptions import ObjectNotFound -from galaxy.util import string_as_bool, umask_fix_perms +from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.util import string_as_bool, umask_fix_perms, safe_relpath from galaxy.util.directory_hash import directory_hash_id from galaxy.util.sleeper import Sleeper from .s3_multipart_upload import multipart_upload @@ -208,6 +208,20 @@ class S3ObjectStore(ObjectStore): umask_fix_perms( path, self.config.umask, 0666, self.config.gid ) def _construct_path(self, obj, base_dir=None, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) rel_path = os.path.join(*directory_hash_id(obj.id)) if extra_dir is not None: if extra_dir_at_root: From 64cd02ae385fb509378c78e6e4832768ff4f1e00 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:22 -0500 Subject: [PATCH 33/35] Remove sample tracking manual external service transfer due to security concerns --- lib/galaxy/model/__init__.py | 32 -- .../galaxy/controllers/requests_admin.py | 326 +----------------- .../requests/select_datasets_to_transfer.mako | 149 -------- .../galaxy/requests/common/common.mako | 12 +- .../requests/common/view_request_history.mako | 1 - .../requests/common/view_sample_datasets.mako | 4 - 6 files changed, 3 insertions(+), 521 deletions(-) delete mode 100644 templates/admin/requests/select_datasets_to_transfer.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 23401ca00a9..926851eff62 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -24,11 +24,6 @@ from sqlalchemy import and_, func, not_, or_, true, join, select from sqlalchemy.orm import joinedload, object_session, aliased from sqlalchemy.ext import hybrid -try: - import pexpect -except ImportError: - pexpect = None - import galaxy.datatypes import galaxy.datatypes.registry import galaxy.model.orm.now @@ -60,9 +55,6 @@ datatypes_registry.load_datatypes() # this be unlimited - filter in Python if over this limit. MAX_IN_FILTER_LENGTH = 100 -PEXPECT_IMPORT_MESSAGE = ('The Python pexpect package is required to use this ' - 'feature, please install it') - class NoConverterException(Exception): def __init__(self, value): @@ -4251,30 +4243,6 @@ class Sample( object, Dictifiable ): untransferred_datasets.append( dataset ) return untransferred_datasets - def get_untransferred_dataset_size( self, filepath, scp_configs ): - def print_ticks( d ): - pass - if pexpect is None: - return PEXPECT_IMPORT_MESSAGE - error_msg = 'Error encountered in determining the file size of %s on the external_service.' % filepath - if not scp_configs['host'] or not scp_configs['user_name'] or not scp_configs['password']: - return error_msg - login_str = '%s@%s' % ( scp_configs['user_name'], scp_configs['host'] ) - cmd = 'ssh %s "du -sh \'%s\'"' % ( login_str, filepath ) - try: - output = pexpect.run( cmd, - events={ '.ssword:*': scp_configs['password'] + '\r\n', - pexpect.TIMEOUT: print_ticks}, - timeout=10 ) - except Exception: - return error_msg - # cleanup the output to get just the file size - return output.replace( filepath, '' )\ - .replace( 'Password:', '' )\ - .replace( "'s password:", '' )\ - .replace( login_str, '' )\ - .strip() - @property def run_details( self ): # self.runs is a list of SampleRunAssociations ordered descending on update_time. diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 827e1020e41..bbde846e6a7 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -10,15 +10,7 @@ from .requests_common import RequestsGrid, invalid_id_redirect from galaxy import eggs eggs.require( "MarkupSafe" ) from markupsafe import escape -eggs.require("amqp") -import amqp -try: - import pexpect -except ImportError: - pexpect = None -PEXPECT_IMPORT_MESSAGE = ('The Python pexpect package is required to use this ' - 'feature, please install it') log = logging.getLogger( __name__ ) @@ -372,207 +364,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): action='manage_datasets', sample_id=sample_id ) ) - @web.expose - @web.require_admin - def select_datasets_to_transfer( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - request_id = kwd.get( 'request_id', None ) - external_service_id = kwd.get( 'external_service_id', None ) - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - # Load the data transfer settings - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - selected_datasets_to_transfer = util.restore_text( params.get( 'selected_datasets_to_transfer', '' ) ) - if selected_datasets_to_transfer: - selected_datasets_to_transfer = selected_datasets_to_transfer.split(',') - else: - selected_datasets_to_transfer = [] - sample_id = kwd.get( 'sample_id', 'none' ) - sample_id_select_field = self.__build_sample_id_select_field( trans, request, sample_id ) - if sample_id != 'none': - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) - else: - sample = None - # The __get_files() method redirects here with a status of 'error' and a message if there - # was a problem retrieving the files. - if params.get( 'select_datasets_to_transfer_button', False ): - # Get the sample that was sequenced to produce these datasets. - if sample_id == 'none': - del kwd[ 'select_datasets_to_transfer_button' ] - message = 'Select the sample that was sequenced to produce the datasets you want to transfer.' - kwd[ 'message' ] = message - kwd[ 'status' ] = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - **kwd ) ) - if not sample.library: - # Display an error if a sample has been selected that - # has not yet been associated with a destination library. - message = 'Select a target data library and folder for the sample before selecting the datasets.' - status = 'error' - return trans.response.send_redirect( web.url_for( controller='requests_common', - action='edit_samples', - cntrller='requests_admin', - id=trans.security.encode_id( request.id ), - status=status, - message=message ) ) - # Save the sample datasets - sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) - if sample_dataset_file_names: - message = 'Datasets (%s) have been selected for sample (%s)' % \ - ( str( sample_dataset_file_names )[1:-1].replace( "'", "" ), sample.name ) - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - request_id=request_id, - sample_id=sample_id, - message=message, - status=status ) ) - return trans.fill_template( '/admin/requests/select_datasets_to_transfer.mako', - cntrller='requests_admin', - request=request, - external_service=external_service, - scp_configs=scp_configs, - sample=sample, - sample_id_select_field=sample_id_select_field, - status=status, - message=message ) - - @web.json - def get_file_details( self, trans, request_id, external_service_id, folder_path ): - def print_ticks( d ): - # pexpect timeout method - pass - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - if pexpect is None: - return PEXPECT_IMPORT_MESSAGE - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - cmd = 'ssh %s@%s "ls -oghp \'%s\'"' % ( scp_configs[ 'user_name' ], - scp_configs[ 'host' ], - folder_path ) - # Handle the authentication message if ssh keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - for password_str in [ 'Password:\r\n', 'password:\r\n' ]: - # Eliminate the output created using ssh from the tree - if password_str in output: - output = output.replace( password_str, '' ) - return unicode( output.replace( '\r\n', '
    ' ) ) - - @web.json - def open_folder( self, trans, request_id, external_service_id, key ): - # Avoid caching - trans.response.headers['Pragma'] = 'no-cache' - trans.response.headers['Expires'] = '0' - request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) - external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) - folder_path = key - files_list = self.__get_files( trans, request, external_service, folder_path ) - folder_contents = [] - for filename in files_list: - is_folder = False - if filename and filename[-1] == os.sep: - is_folder = True - if filename: - full_path = os.path.join( folder_path, filename ) - node = { "title": filename, - "isFolder": is_folder, - "isLazy": is_folder, - "tooltip": full_path, - "key": full_path } - folder_contents.append( node ) - return folder_contents - - def __get_files( self, trans, request, external_service, folder_path ): - # Retrieves the filenames to be transferred from the remote host. - ok = True - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - if not scp_configs[ 'host' ] or not scp_configs[ 'user_name' ] or not scp_configs[ 'password' ]: - status = 'error' - message = "Error in external service login information." - ok = False - - def print_ticks( d ): - pass - cmd = 'ssh %s@%s "ls -p \'%s\'"' % ( scp_configs[ 'user_name' ], scp_configs[ 'host' ], folder_path ) - # Handle the authentication message if keys are not set - the message is - # something like: "Are you sure you want to continue connecting (yes/no)." - if pexpect is not None: - output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', - '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, - timeout=10 ) - if 'No such file or directory' in output: - status = 'error' - message = "No folder named (%s) exists on the external service." % folder_path - ok = False - else: - status = 'error' - message = PEXPECT_IMPORT_MESSAGE - ok = False - if ok: - if 'assword:' in output: - # Eliminate the output created using ssh from the tree - output_as_list = output.splitlines()[ 1: ] - else: - output_as_list = output.splitlines() - return output_as_list - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='select_datasets_to_transfer', - request_id=trans.security.encode_id( request.id ), - external_service_id=trans.security.encode_id( external_service.id ), - status=status, - message=message ) ) - - def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - sample_dataset_file_names = [] - if selected_datasets_to_transfer: - for filepath in selected_datasets_to_transfer: - # FIXME: handle folder selection - ignore folders for now - if filepath[-1] != os.sep: - name = self.__rename_dataset( sample, filepath.split( '/' )[-1], scp_configs ) - status = trans.app.model.SampleDataset.transfer_status.NOT_STARTED - size = sample.get_untransferred_dataset_size( filepath, scp_configs ) - sample_dataset = trans.model.SampleDataset( sample=sample, - file_path=filepath, - status=status, - name=name, - error_msg='', - size=size, - external_service=external_service ) - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - sample_dataset_file_names.append( str( sample_dataset.name ) ) - return sample_dataset_file_names - - def __rename_dataset( self, sample, filepath, scp_configs ): - name = filepath.split( '/' )[-1] - options = sample.request.type.rename_dataset_options - option = scp_configs.get( 'rename_dataset', options.NO ) - if option == options.SAMPLE_NAME: - new_name = sample.name + '_' + name - if option == options.EXPERIMENT_AND_SAMPLE_NAME: - new_name = sample.request.name + '_' + sample.name + '_' + name - if option == options.EXPERIMENT_NAME: - new_name = sample.request.name + '_' + name - else: - new_name = name - return util.sanitize_for_filename( new_name ) - def __ensure_library_add_permission( self, trans, target_library, target_folder ): """ Ensures the current admin user has ADD_LIBRARY permission on the target data library and folder. @@ -595,92 +386,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): if flush_needed: trans.sa_session.flush() - def __create_data_transfer_messages( self, trans, sample, selected_sample_datasets ): - """ - Creates the xml messages to send to the rabbitmq server. It returns a dictionary of messages - keyed by the external service used to transfer the datasets - """ - # Create the xml message based on the following template - xml = \ - ''' - %(GALAXY_HOST)s - %(API_KEY)s - %(DATA_HOST)s - %(DATA_USER)s - %(DATA_PASSWORD)s - %(REQUEST_ID)s - %(SAMPLE_ID)s - %(LIBRARY_ID)s - %(FOLDER_ID)s - %(DATASETS)s - ''' - dataset_xml = \ - ''' - %(ID)s - %(NAME)s - %(FILE)s - ''' - # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and - # send a single AMQP message to the galaxy_listener - dataset_elements = {} - for sample_dataset in selected_sample_datasets: - external_service = sample_dataset.external_service - if sample_dataset.status == trans.app.model.SampleDataset.transfer_status.NOT_STARTED: - if external_service not in dataset_elements: - dataset_elements[ external_service ] = '' - dataset_elements[ external_service ] += dataset_xml % dict( ID=str( sample_dataset.id ), - NAME=sample_dataset.name, - FILE=sample_dataset.file_path ) - # update the dataset transfer status - sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE - trans.sa_session.add( sample_dataset ) - trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets - messages = [] - for external_service, dataset_elem in dataset_elements.items(): - external_service.load_data_transfer_settings( trans ) - scp_configs = external_service.data_transfer[ trans.model.ExternalService.data_transfer_protocol.SCP ] - # Check data transfer settings - err_msg = self.__validate_data_transfer_settings( trans, sample.request.type, scp_configs ) - if err_msg: - return trans.response.send_redirect( web.url_for( controller='requests_admin', - action='manage_datasets', - sample_id=trans.security.encode_id( sample.id ), - status='error', - message=err_msg ) ) - message = xml % dict( GALAXY_HOST=trans.request.host, - API_KEY=trans.user.api_keys[0].key, - DATA_HOST=scp_configs[ 'host' ], - DATA_USER=scp_configs[ 'user_name' ], - DATA_PASSWORD=scp_configs[ 'password' ], - REQUEST_ID=str( sample.request.id ), - SAMPLE_ID=str( sample.id ), - LIBRARY_ID=str( sample.library.id ), - FOLDER_ID=str( sample.folder.id ), - DATASETS=dataset_elem ) - messages.append( message.replace( '\n', '' ).replace( '\r', '' ) ) - return messages - - def __validate_data_transfer_settings( self, trans, request_type, scp_configs ): - err_msg = '' - # check the external service login info - if not scp_configs.get( 'host', '' ) or \ - not scp_configs.get( 'user_name', '' ) or \ - not scp_configs.get( 'password', '' ): - err_msg += "Error in external service login information. " - if not trans.user.api_keys: - err_msg += "Set your API Key in your User Preferences to transfer datasets. " - # Check if library_import_dir is set - if not trans.app.config.library_import_dir: - err_msg = "'The library_import_dir' setting is not correctly set in the Galaxy config file. " - # Check the RabbitMQ server settings in the config file - for k, v in trans.app.config.amqp.items(): - if not v: - err_msg += 'Set RabbitMQ server settings in the "galaxy_amqp" section of the Galaxy config file, specifically "%s" is not set.' % k - break - return err_msg - @web.expose @web.require_admin def initiate_data_transfer( self, trans, sample_id, sample_datasets=[], sample_dataset_id='' ): @@ -722,35 +427,8 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service=external_service, external_service_type=external_service_type ) else: - # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we - # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform - # constant db hits to check for deferred jobs that are not in a finished state. - # Create the message - messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages - for rmq_msg in messages: - try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ]) - chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', - application_headers={ 'msg_type': 'data_transfer' } ) - msg.properties[ "delivery_mode" ] = 2 - chan.basic_publish( msg, - exchange=trans.app.config.amqp[ 'exchange' ], - routing_key=trans.app.config.amqp[ 'routing_key' ] ) - chan.close() - conn.close() - except Exception, e: - message = "Error sending the data transfer message to the Galaxy AMQP message queue:
    %s" % str(e) - status = "error" - if not message: - message = "%i datasets have been queued for transfer from the external service." % len( sample_datasets ) - status = "done" + message = "Message queue transfer is no longer supported, please set enable_beta_job_managers = True in galaxy.ini" + status = "error" return trans.response.send_redirect( web.url_for( controller='requests_admin', action='manage_datasets', sample_id=trans.security.encode_id( sample.id ), diff --git a/templates/admin/requests/select_datasets_to_transfer.mako b/templates/admin/requests/select_datasets_to_transfer.mako deleted file mode 100644 index 10d2135b7ef..00000000000 --- a/templates/admin/requests/select_datasets_to_transfer.mako +++ /dev/null @@ -1,149 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> -<%namespace file="/requests/common/common.mako" import="render_sample_datasets" /> -<%namespace file="/requests/common/common.mako" import="common_javascripts" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - ${common_javascripts()} - - -${h.js( "libs/jquery/jquery-ui", "libs/jquery/jquery.cookie", "libs/jquery/jquery.dynatree" )} -${h.css( "dynatree_skin/ui.dynatree" )} - - - -<% - is_admin = cntrller == 'requests_admin' and trans.user_is_admin() - can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder -%> - -

    - - -%if not sample: -
    - Select a sample before selecting datasets to transfer -
    -%endif - -%if request.samples_without_library_destinations: -

    - Select a target data library and folder for a sample before selecting its datasets to transfer from the external service -

    -%endif - -%if message: - ${render_msg( message, status )} -%endif - -
    -
    Select datasets to transfer from data directory configured for the external service
    -
    -
    - - ${sample_id_select_field.get_html()} -
    - Select the sample that was sequenced to produce the datasets you want to transfer. -
    -
    -
    - -
    - Loading... -
    - -
    -
      -
    • Click the external service configuration button and change the Data directory setting to redefine the source data location.
      • -
    • Select a folder to select all of the individual files within that folder.
      • -
    • Click the Select datasets button when desired dataset check boxes are checked.
      • -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    - -%if sample and sample.datasets: - <% title = 'All selected datasets for "%s"' % sample.name %> -

    - ${render_sample_datasets( 'requests_admin', sample, sample.datasets, title )} -%endif diff --git a/templates/webapps/galaxy/requests/common/common.mako b/templates/webapps/galaxy/requests/common/common.mako index 216b7165cbf..28808007832 100644 --- a/templates/webapps/galaxy/requests/common/common.mako +++ b/templates/webapps/galaxy/requests/common/common.mako @@ -333,7 +333,6 @@ can_add_samples = is_unsubmitted can_delete_samples = not adding_new_samples and request.samples and ( ( is_admin and not is_complete ) or is_unsubmitted ) can_edit_samples = request.samples and ( is_admin or not is_complete ) - can_select_datasets = is_admin and displayable_sample_widgets and ( is_submitted or is_complete ) can_transfer_datasets = is_admin and request.samples and not request.is_rejected display_checkboxes = not adding_new_samples and ( is_complete or is_rejected or is_submitted ) display_bar_code = request.samples and ( is_complete or is_rejected or is_submitted ) @@ -407,7 +406,7 @@ %elif sample: - %if sample.state and ( can_select_datasets or can_transfer_datasets ): + %if sample.state and can_transfer_datasets: ## A sample will have a state only after the request has been submitted. <% encoded_id = trans.security.encode_id( sample.id ) @@ -419,14 +418,6 @@ ${sample.name | h}

    - %if can_select_datasets: - %for external_service in sample.request.type.get_external_services_for_manual_data_transfer( trans ): - <% - menu_item_label = "Select datasets to transfer using %s" % external_service.name - %> -
  • ${menu_item_label}
    • - %endfor - %endif %if sample.datasets and len( sample.datasets ) > len( transferred_dataset_files ) and sample.library and sample.folder:
  • Manage selected datasets
    • %elif sample.datasets and len( sample.datasets ) == len( transferred_dataset_files ): @@ -665,7 +656,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files %> ## The transfer status should update only when the request has been submitted or complete diff --git a/templates/webapps/galaxy/requests/common/view_request_history.mako b/templates/webapps/galaxy/requests/common/view_request_history.mako index 89faaaafd48..913914ddf80 100644 --- a/templates/webapps/galaxy/requests/common/view_request_history.mako +++ b/templates/webapps/galaxy/requests/common/view_request_history.mako @@ -11,7 +11,6 @@ can_add_samples = is_unsubmitted can_edit_request = ( is_admin and not is_complete ) or is_unsubmitted can_reject = is_admin and is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_submit_request = request.samples and is_unsubmitted %> diff --git a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako index 986c88d78a9..9f4210f9ed5 100644 --- a/templates/webapps/galaxy/requests/common/view_sample_datasets.mako +++ b/templates/webapps/galaxy/requests/common/view_sample_datasets.mako @@ -12,7 +12,6 @@ is_admin = cntrller == 'requests_admin' and trans.user_is_admin() is_complete = sample.request.is_complete is_submitted = sample.request.is_submitted - can_select_datasets = is_admin and ( is_complete or is_submitted ) can_transfer_datasets = is_admin and sample.untransferred_dataset_files and sample.library and sample.folder %> @@ -24,9 +23,6 @@ %endif
  • Dataset Actions
    • - %if can_select_datasets: -
  • Select more datasets
    • - %endif
  • View target Data Library
  • Browse this request
    • From 2dcbda93b29e4dd975453af8eeb4f171f7db2918 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:22 -0500 Subject: [PATCH 34/35] Security fixes for tool shed repository browsing --- .../galaxy/controllers/admin_toolshed.py | 8 ++-- .../tool_shed/controllers/repository.py | 8 ++-- lib/tool_shed/util/shed_util_common.py | 46 +++++++++++++++---- .../admin/tool_shed_repository/common.mako | 12 +++-- .../webapps/tool_shed/repository/common.mako | 10 ++-- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index cc947b569c3..345d98f10f1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -390,11 +390,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose @web.require_admin @@ -918,11 +918,11 @@ class AdminToolshed( AdminGalaxy ): @web.json @web.require_admin - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 411f420ff7a..eb25b47142a 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1470,11 +1470,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def get_file_contents( self, trans, file_path ): + def get_file_contents( self, trans, file_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.get_repository_file_contents( file_path ) + return suc.get_repository_file_contents( trans.app, file_path, repository_id ) @web.expose def get_functional_test_rss( self, trans, **kwd ): @@ -2439,11 +2439,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return '' @web.json - def open_folder( self, trans, folder_path ): + def open_folder( self, trans, folder_path, repository_id ): # Avoid caching trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' - return suc.open_repository_files_folder( folder_path ) + return suc.open_repository_files_folder( trans.app, folder_path, repository_id ) @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 7c6f1d8b2a3..08ef31f81d6 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -600,9 +600,17 @@ def get_repository_for_dependency_relationship( app, tool_shed, name, owner, cha return repository -def get_repository_file_contents( file_path ): +def get_repository_file_contents( app, file_path, repository_id ): """Return the display-safe contents of a repository file for display in a browser.""" - if checkers.is_gzip( file_path ): + safe_str = '' + if not is_path_within_repo( app, file_path, repository_id ): + log.warning( 'Request tries to access a file outside of the repository location. File path: %s', file_path ) + return 'Invalid file path' + # Symlink targets are checked by is_path_within_repo + if os.path.islink( file_path ): + safe_str = 'link to: ' + basic_util.to_html_string( os.readlink( file_path ) ) + return safe_str + elif checkers.is_gzip( file_path ): return '
    gzip compressed file
    ' elif checkers.is_bz2( file_path ): return '
    bz2 compressed file
    ' @@ -611,7 +619,6 @@ def get_repository_file_contents( file_path ): elif checkers.check_binary( file_path ): return '
    Binary file
    ' else: - safe_str = '' for i, line in enumerate( open( file_path ) ): safe_str = '%s%s' % ( safe_str, basic_util.to_html_string( line ) ) # Stop reading after string is larger than MAX_CONTENT_SIZE. @@ -621,6 +628,7 @@ def get_repository_file_contents( file_path ): util.nice_size( MAX_CONTENT_SIZE ) safe_str = '%s%s' % ( safe_str, large_str ) break + if len( safe_str ) > basic_util.MAX_DISPLAY_SIZE: # Eliminate the middle of the file to display a file no larger than basic_util.MAX_DISPLAY_SIZE. # This may not be ideal if the file is larger than MAX_CONTENT_SIZE. @@ -642,9 +650,6 @@ def get_repository_files( folder_path ): # Skip .hg directories if item.startswith( '.hg' ): continue - if os.path.isdir( os.path.join( folder_path, item ) ): - # Append a '/' character so that our jquery dynatree will function properly. - item = '%s/' % item contents.append( item ) if contents: contents.sort() @@ -1109,11 +1114,15 @@ def is_tool_shed_client( app ): return hasattr( app, "install_model" ) -def open_repository_files_folder( folder_path ): +def open_repository_files_folder( app, folder_path, repository_id ): """ Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy. """ + # Symlink targets are checked by is_path_within_repo + if not is_path_within_repo( app, folder_path, repository_id ): + log.warning( 'Request tries to access a folder outside of the repository location. Folder path: %s', folder_path ) + return [] try: files_list = get_repository_files( folder_path ) except OSError, e: @@ -1123,10 +1132,17 @@ def open_repository_files_folder( folder_path ): folder_contents = [] for filename in files_list: is_folder = False - if filename and filename[ -1 ] == os.sep: - is_folder = True + full_path = os.path.join( folder_path, filename ) + is_link = os.path.islink( full_path ) + path_is_within_repo = is_path_within_repo( app, full_path, repository_id ) + if is_link and not path_is_within_repo: + log.warning( 'Valid folder contains a symlink outside of the repository location. Link found in: ' + str( full_path ) ) if filename: - full_path = os.path.join( folder_path, filename ) + if os.path.isdir( full_path ) and path_is_within_repo: + # Append a '/' character so that our jquery dynatree will function properly. + filename = '%s/' % filename + full_path = '%s/' % full_path + is_folder = True node = { "title": filename, "isFolder": is_folder, "isLazy": is_folder, @@ -1136,6 +1152,16 @@ def open_repository_files_folder( folder_path ): return folder_contents +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folde ron the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path + + def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): """ Find out if a repository is already installed into Galaxy - there are several scenarios where this diff --git a/templates/admin/tool_shed_repository/common.mako b/templates/admin/tool_shed_repository/common.mako index 6cf501e1baa..ec06a65a6c3 100644 --- a/templates/admin/tool_shed_repository/common.mako +++ b/templates/admin/tool_shed_repository/common.mako @@ -23,14 +23,16 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${directory_path|h}" }, + dataType: "json", + data: { folder_path: "${directory_path|h}", + repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", + url: "${h.url_for( controller='admin_toolshed', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, + repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -55,7 +57,7 @@ type: "POST", url: "${h.url_for( controller='admin_toolshed', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function( data ) { cell.html( '' ) } diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index cd892fdc74a..4e67c86127d 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -22,14 +22,14 @@ fx: { height: "toggle", duration: 200 }, // initAjax is hard to fake, so we pass the children as object array: initAjax: {url: "${h.url_for( controller='repository', action='open_folder' )}", - dataType: "json", - data: { folder_path: "${repository.repo_path( trans.app )}" }, + dataType: "json", + data: { folder_path: "${repository.repo_path( trans.app )}", repository_id: "${trans.security.encode_id( repository.id )}" }, }, onLazyRead: function(dtnode){ dtnode.appendAjax({ - url: "${h.url_for( controller='repository', action='open_folder' )}", + url: "${h.url_for( controller='repository', action='open_folder' )}", dataType: "json", - data: { folder_path: dtnode.data.key }, + data: { folder_path: dtnode.data.key, repository_id: "${trans.security.encode_id( repository.id )}" }, }); }, onSelect: function(select, dtnode) { @@ -62,7 +62,7 @@ type: "POST", url: "${h.url_for( controller='repository', action='get_file_contents' )}", dataType: "json", - data: { file_path: selected_value }, + data: { file_path: selected_value, repository_id: "${trans.security.encode_id( repository.id )}" }, success : function ( data ) { cell.html( '' ) } From b11b8e68500a3d074bf20b08958671d958d42eca Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Feb 2016 11:18:22 -0500 Subject: [PATCH 35/35] Security fixes for tool shed hg push and capsule/tarball uploads --- .../tool_shed/framework/middleware/hg.py | 21 ++++++- lib/tool_shed/capsule/capsule_manager.py | 57 ++++++++++++------- lib/tool_shed/util/commit_util.py | 54 +++++++++++------- lib/tool_shed/util/repository_content_util.py | 26 +++------ 4 files changed, 99 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 184710662ed..7cc2c0bdfd6 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -10,7 +10,7 @@ from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import AUTH_TYPE from paste.httpheaders import REMOTE_USER -from galaxy.util import asbool +from galaxy.util import asbool, safe_relpath from galaxy.util.hash_util import new_secure_hash from tool_shed.util import hg_util from tool_shed.util import commit_util @@ -115,7 +115,11 @@ class Hg( object ): fh.write( chunk ) fh.close() fh = open( tmp_filename, 'rb' ) - changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + try: + changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) + except AttributeError: + msg = 'Your version of Mercurial is not supported. Please use a version < 3.5' + return self.__display_exception_remotely( start_response, msg ) fh.close() try: os.unlink( tmp_filename ) @@ -124,6 +128,19 @@ class Hg( object ): if changeset_groups: # Check the repository type to make sure inappropriate files are not being pushed. if 'PATH_INFO' in environ: + # Ensure there are no symlinks with targets outside the repo + for entry in changeset_groups: + if len( entry ) == 2: + filename, change_list = entry + if not isinstance(change_list, list): + change_list = [change_list] + for change in change_list: + for patch in change['data']: + target = patch['block'].strip() + if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ): + msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target ) + log.warning( msg ) + return self.__display_exception_remotely( start_response, msg ) # Instantiate a database connection engine = sqlalchemy.create_engine( self.db_url ) connection = engine.connect() diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 76011806757..7a8a0d60a72 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -16,6 +16,7 @@ import tool_shed.repository_types.util as rt_util from galaxy import web from galaxy.util import asbool from galaxy.util import CHUNK_SIZE +from galaxy.util import safe_relpath from galaxy.util.odict import odict from tool_shed.dependencies.repository.relation_builder import RelationBuilder from tool_shed.dependencies import attribute_handlers @@ -721,29 +722,16 @@ class ImportRepositoryManager( object ): hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, error_message = commit_util.check_archive( repository, archive ) - if ok: + check_results = commit_util.check_archive( repository, archive ) + # We filter out undesirable files but fail on undesriable dirs. Not + # sure why, just trying to maintain the same behavior as before. -nate + if not check_results.invalid and not check_results.undesirable_dirs: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in archive.getmembers(): - # Check files and directories in the archive. - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - error_message = 'Import failed: invalid file path %s in archive %s' % \ - ( str( file_path_item ), str( archive_file_name ) ) - results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message - return results_dict - filenames_in_archive.append( tarinfo_obj.name ) - else: - undesirable_files_removed += 1 # Extract the uploaded archive to the repository root. - archive.extractall( path=full_path ) + archive.extractall( path=full_path, members=check_results.valid ) archive.close() - for filename in filenames_in_archive: + for tar_member in check_results.valid: + filename = tar_member.name uploaded_file_name = os.path.join( full_path, filename ) if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: # Inspect the contents of the file to see if toolshed or changeset_revision attributes @@ -770,6 +758,9 @@ class ImportRepositoryManager( object ): new_repo_alert = True # Since the repository is new, the following must be False. remove_repo_files_not_in_tar = False + filenames_in_archive = [ member.name for member in check_results.valid ] + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = 0 ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \ commit_util.handle_directory_changes( self.app, self.host, @@ -800,7 +791,13 @@ class ImportRepositoryManager( object ): else: archive.close() results_dict[ 'ok' ] = False - results_dict[ 'error_message' ] += error_message + results_dict[ 'error_message' ] += 'Capsule errors were found: ' + if check_results.invalid: + results_dict[ 'error_message' ] += '%s Invalid files were: %s.' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + if check_results.undesirable_dirs: + results_dict[ 'error_message' ] += ' Undesirable directories were: %s.' % ( + ', '.join( check_results.undesirable_dirs ) ) return results_dict def upload_capsule( self, **kwd ): @@ -853,6 +850,12 @@ class ImportRepositoryManager( object ): return_dict[ 'status' ] = 'error' uploaded_file.close() return return_dict + if not self.validate_archive_paths( tar_archive ): + return_dict[ 'status' ] = 'error' + return_dict[ 'message' ] = ( 'This capsule contains an invalid member type ' + 'or a file outside the archive path.' ) + uploaded_file.close() + return return_dict return_dict[ 'tar_archive' ] = tar_archive return_dict[ 'capsule_file_name' ] = uploaded_file_filename uploaded_file.close() @@ -862,6 +865,18 @@ class ImportRepositoryManager( object ): return return_dict return return_dict + def validate_archive_paths( self, tar_archive ): + ''' + Inspect the archive contents to ensure that there are no risky symlinks. + Returns True if a suspicious path is found. + ''' + for member in tar_archive.getmembers(): + if not ( member.isdir() or member.isfile() or member.islnk() ): + return False + elif not safe_relpath( member.name ): + return False + return True + def validate_capsule( self, **kwd ): """ Inspect the uploaded capsule's manifest and its contained files to ensure it is a valid diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 9628d1c3f9e..b740ab05e5d 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -5,6 +5,7 @@ import logging import os import shutil import tempfile +from collections import namedtuple from galaxy import eggs eggs.require('SQLAlchemy') @@ -12,6 +13,7 @@ from sqlalchemy.sql.expression import null import tool_shed.repository_types.util as rt_util from galaxy.datatypes import checkers +from galaxy.util import safe_relpath from tool_shed.tools import data_table_manager from tool_shed.util import basic_util, hg_util, shed_util_common as suc @@ -22,30 +24,44 @@ UNDESIRABLE_FILES = [ '.hg_archival.txt', 'hgrc', '.DS_Store', 'tool_test_output def check_archive( repository, archive ): + valid = [] + invalid = [] + errors = [] + undesirable_files = [] + undesirable_dirs = [] for member in archive.getmembers(): # Allow regular files and directories only if not ( member.isdir() or member.isfile() or member.islnk() ): - message = "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc). " - message += "The problematic member in this archive is %s," % str( member.name ) - return False, message - for item in [ '.hg', '..', '/' ]: - if member.name.startswith( item ): - message = "Uploaded archives cannot contain .hg directories, absolute filenames starting with '/', or filenames with two dots '..'. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message - if member.name in [ 'hgrc' ]: - message = "Uploaded archives cannot contain hgrc files. " - message += "The problematic member in this archive is %s." % str( member.name ) - return False, message + errors.append( "Uploaded archives can only include regular directories and files (no symbolic links, devices, etc)." ) + invalid.append( member ) + continue + if not safe_relpath( member.name ): + errors.append( "Uploaded archives cannot contain files that would extract outside of the archive." ) + invalid.append( member ) + continue + if os.path.basename( member.name ) in UNDESIRABLE_FILES: + undesirable_files.append( member ) + continue + head = tail = member.name + try: + while tail: + head, tail = os.path.split(head) + if tail in UNDESIRABLE_DIRS: + undesirable_dirs.append( member ) + assert False + except AssertionError: + continue if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and member.name != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message + errors.append( 'Repositories of type Repository suite definition can contain only a single file named repository_dependencies.xml.' ) + invalid.append( member ) + continue if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and member.name != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - message = 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' - message += 'This archive contains a member named %s.' % str( member.name ) - return False, message - return True, '' + errors.append( 'Repositories of type Tool dependency definition can contain only a single file named tool_dependencies.xml.' ) + invalid.append( member ) + continue + valid.append( member ) + ArchiveCheckResults = namedtuple( 'ArchiveCheckResults', [ 'valid', 'invalid', 'undesirable_files', 'undesirable_dirs', 'errors' ] ) + return ArchiveCheckResults( valid, invalid, undesirable_files, undesirable_dirs, errors ) def check_file_contents_for_email_alerts( app ): diff --git a/lib/tool_shed/util/repository_content_util.py b/lib/tool_shed/util/repository_content_util.py index 99fc9793b83..1cf6660278c 100644 --- a/lib/tool_shed/util/repository_content_util.py +++ b/lib/tool_shed/util/repository_content_util.py @@ -15,31 +15,23 @@ def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) undesirable_dirs_removed = 0 undesirable_files_removed = 0 - ok, message = commit_util.check_archive( repository, tar ) - if not ok: + check_results = commit_util.check_archive( repository, tar ) + if check_results.invalid: tar.close() uploaded_file.close() - return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed + message = '%s Invalid paths were: %s' % ( + ' '.join( check_results.errors ), ', '.join( check_results.invalid ) ) + return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) else: full_path = os.path.abspath( repo_dir ) - filenames_in_archive = [] - for tarinfo_obj in tar.getmembers(): - ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES - if ok: - for file_path_item in tarinfo_obj.name.split( '/' ): - if file_path_item in commit_util.UNDESIRABLE_DIRS: - undesirable_dirs_removed += 1 - ok = False - break - else: - undesirable_files_removed += 1 - if ok: - filenames_in_archive.append( tarinfo_obj.name ) + undesirable_files_removed = len( check_results.undesirable_files ) + undesirable_dirs_removed = len( check_results.undesirable_dirs ) + filenames_in_archive = [ ti.name for ti in check_results.valid ] # Extract the uploaded tar to the load_point within the repository hierarchy. - tar.extractall( path=full_path ) + tar.extractall( path=full_path, members=check_results.valid ) tar.close() uploaded_file.close() for filename in filenames_in_archive: