From 47b63cbf9121a4b0591aa4cd67cd268030624419 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Tue, 14 Aug 2012 10:04:25 +1000 Subject: [PATCH] Add the ability for Galaxy's ObjectStore to use OpenStack's SWIFT object store as the backend data storage --- lib/galaxy/config.py | 12 ++++++--- lib/galaxy/objectstore/__init__.py | 39 ++++++++++++++++++++++++------ universe_wsgi.ini.sample | 22 +++++++++++------ 3 files changed, 53 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e0893cb2c43..c55516ec843 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -169,10 +169,14 @@ class Configuration( object ): if self.nginx_upload_store: self.nginx_upload_store = os.path.abspath( self.nginx_upload_store ) self.object_store = kwargs.get( 'object_store', 'disk' ) - self.aws_access_key = kwargs.get( 'aws_access_key', None ) - self.aws_secret_key = kwargs.get( 'aws_secret_key', None ) - self.s3_bucket = kwargs.get( 's3_bucket', None) - self.use_reduced_redundancy = kwargs.get( 'use_reduced_redundancy', False ) + self.os_access_key = kwargs.get( 'os_access_key', None ) + self.os_secret_key = kwargs.get( 'os_secret_key', None ) + self.os_bucket_name = kwargs.get( 'os_bucket_name', None ) + self.os_host = kwargs.get( 'os_host', None ) + self.os_port = kwargs.get( 'os_port', None ) + self.os_is_secure = string_as_bool( kwargs.get( 'os_is_secure', True ) ) + self.os_conn_path = kwargs.get( 'os_conn_path', '/' ) + self.os_use_reduced_redundancy = kwargs.get( 'os_use_reduced_redundancy', False ) self.object_store_cache_size = float(kwargs.get( 'object_store_cache_size', -1 )) self.distributed_object_store_config_file = kwargs.get( 'distributed_object_store_config_file', None ) # Parse global_conf and save the parser diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 6af4414ac8c..ab2fe8f8aae 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -25,6 +25,7 @@ from sqlalchemy.orm import object_session if sys.version_info >= (2, 6): import multiprocessing from galaxy.objectstore.s3_multipart_upload import multipart_upload + import boto from boto.s3.key import Key from boto.s3.connection import S3Connection from boto.exception import S3ResponseError @@ -377,9 +378,9 @@ class S3ObjectStore(ObjectStore): super(S3ObjectStore, self).__init__() self.config = config self.staging_path = self.config.file_path - self.s3_conn = S3Connection() - self.bucket = self._get_bucket(self.config.s3_bucket) - self.use_rr = self.config.use_reduced_redundancy + self.s3_conn = get_OS_connection(self.config) + self.bucket = self._get_bucket(self.config.os_bucket_name) + self.use_rr = self.config.os_use_reduced_redundancy self.cache_size = self.config.object_store_cache_size self.transfer_progress = 0 # Clean cache only if value is set in universe_wsgi.ini @@ -468,7 +469,7 @@ class S3ObjectStore(ObjectStore): for i in range(5): try: bucket = self.s3_conn.get_bucket(bucket_name) - log.debug("Using S3 object store; got bucket '%s'" % bucket.name) + log.debug("Using cloud object store with bucket '%s'" % bucket.name) return bucket except S3ResponseError: log.debug("Could not get bucket '%s', attempt %s/5" % (bucket_name, i+1)) @@ -843,7 +844,6 @@ class S3ObjectStore(ObjectStore): def get_store_usage_percent(self): return 0.0 - class DistributedObjectStore(ObjectStore): """ ObjectStore that defers to a list of backends, for getting objects the @@ -1009,14 +1009,14 @@ def build_object_store_from_config(config): store = config.object_store if store == 'disk': return DiskObjectStore(config=config) - elif store == 's3': - os.environ['AWS_ACCESS_KEY_ID'] = config.aws_access_key - os.environ['AWS_SECRET_ACCESS_KEY'] = config.aws_secret_key + elif store == 's3' or store == 'swift': return S3ObjectStore(config=config) elif store == 'distributed': return DistributedObjectStore(config=config) elif store == 'hierarchical': return HierarchicalObjectStore() + else: + log.error("Unrecognized object store definition: {0}".format(store)) def convert_bytes(bytes): """ A helper function used for pretty printing disk usage """ @@ -1039,3 +1039,26 @@ def convert_bytes(bytes): else: size = '%.2fb' % bytes return size + +def get_OS_connection(config): + """ + Get a connection object for a cloud Object Store specified in the config. + Currently, this is a ``boto`` connection object. + """ + log.debug("Getting a connection object for '{0}' object store".format(config.object_store)) + a_key = config.os_access_key + s_key = config.os_secret_key + if config.object_store == 's3': + return S3Connection(a_key, s_key) + else: + # Establish the connection now + calling_format = boto.s3.connection.OrdinaryCallingFormat() + s3_conn = boto.connect_s3(aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=config.os_is_secure, + host=config.os_host, + port=int(config.os_port), + calling_format=calling_format, + path=config.os_conn_path) + return s3_conn + diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index abb17224c1e..4e627e5ac4c 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -481,16 +481,22 @@ use_interactive = True # -- Beta features -# Object store mode (valid options are: disk, s3, distributed, hierarchical) +# Object store mode (valid options are: disk, s3, swift, distributed, hierarchical) #object_store = disk -#aws_access_key = -#aws_secret_key = -#s3_bucket = -#use_reduced_redundancy = True - +#os_access_key = +#os_secret_key = +#os_bucket_name = +# If using 'swift' object store, you must specify the following connection properties +#os_host = swift.rc.nectar.org.au +#os_port = 8888 +#os_is_secure = False +#os_conn_path = / +# Reduced redundancy can be used only with the 's3' object store +#os_use_reduced_redundancy = False # Size (in GB) that the cache used by object store should be limited to. -# If the value is not specified, the cache size will be limited only by the file -# system size. +# If the value is not specified, the cache size will be limited only by the +# file system size. The file system location of the cache is considered the +# configuration of the ``file_path`` directive defined above. #object_store_cache_size = 100 # Configuration file for the distributed object store, if object_store =