From 0516279b7c5a5b6c11118017517ec06e4e3745d6 Mon Sep 17 00:00:00 2001 From: vahid Date: Thu, 20 Jul 2017 22:06:12 -0700 Subject: [PATCH 001/208] Added CloudBridge to the requirements list. --- lib/galaxy/dependencies/conditional-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 173a16fecb6..565ce7dd23f 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -13,6 +13,7 @@ graphitesend azure-storage==0.32.0 # PyRods not in PyPI python-ldap==2.4.27 +cloudbridge==0.3.1 # Synnefo / Pithos+ object store client kamaki From aaad00162f3ea84798031de798fe99fb0fafe05b Mon Sep 17 00:00:00 2001 From: vahid Date: Thu, 20 Jul 2017 23:00:34 -0700 Subject: [PATCH 002/208] Added CloudBridge to the dependencies check. --- lib/galaxy/dependencies/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 5d950c3fa16..94cd8099e0e 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -109,6 +109,9 @@ class ConditionalDependencies( object ): def check_azure_storage( self ): return 'azure_blob' in self.object_stores + def check_cloudbridge( self ): + return 's3' in self.object_stores + def check_kamaki(self): return 'pithos' in self.object_stores From 55c976cb85237bb0906379d692e212f05a327c2b Mon Sep 17 00:00:00 2001 From: vahid Date: Fri, 21 Jul 2017 22:23:07 -0700 Subject: [PATCH 003/208] (1) Added Cloud to ObjectStore; (2) updated CloudBridge dependencies. --- lib/galaxy/dependencies/__init__.py | 2 +- lib/galaxy/objectstore/cloud.py | 615 ++++++++++++++++++++++++++++ 2 files changed, 616 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/objectstore/cloud.py diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 94cd8099e0e..10ebb8cdd20 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -110,7 +110,7 @@ class ConditionalDependencies( object ): return 'azure_blob' in self.object_stores def check_cloudbridge( self ): - return 's3' in self.object_stores + return 'cloud' in self.object_stores def check_kamaki(self): return 'pithos' in self.object_stores diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py new file mode 100644 index 00000000000..809e92ad95a --- /dev/null +++ b/lib/galaxy/objectstore/cloud.py @@ -0,0 +1,615 @@ +""" +Object Store plugin for Cloud storage. +""" + +import logging +import multiprocessing +import os +import shutil +import subprocess +import threading +import time + +from datetime import datetime + +from galaxy.exceptions import ObjectInvalid, ObjectNotFound +from galaxy.util import ( + directory_hash_id, + safe_relpath, + string_as_bool, + umask_fix_perms, +) +from galaxy.util.sleeper import Sleeper + +from ..objectstore import convert_bytes, ObjectStore +from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList + +try: + # Imports are done this way to allow objectstore code to be used outside of Galaxy. + import boto + + from boto.exception import S3ResponseError + from boto.s3.key import Key + from boto.s3.connection import S3Connection +except ImportError: + boto = None + +NO_BOTO_ERROR_MESSAGE = ("Cloud object store is configured, but no boto dependency available." + "Please install and properly configure boto or modify object store configuration.") + +log = logging.getLogger( __name__ ) +logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy + + +class Cloud( ObjectStore ): + """ + Object store that stores objects as items in an cloud storage. A local + cache exists that is used as an intermediate location for files between + Galaxy and the cloud storage. + """ + def __init__( self, config, config_xml ): + super( Cloud, self ).__init__( config ) + self.staging_path = self.config.file_path + self.transfer_progress = 0 + self._parse_config_xml( config_xml ) + self._configure_connection() + self.bucket = self._get_bucket(self.bucket) + # Clean cache only if value is set in galaxy.ini + if self.cache_size != -1: + # Convert GBs to bytes for comparison + self.cache_size = self.cache_size * 1073741824 + # Helper for interruptable sleep + self.sleeper = Sleeper() + self.cache_monitor_thread = threading.Thread(target=self.__cache_monitor) + self.cache_monitor_thread.start() + log.info("Cache cleaner manager started") + # Test if 'axel' is available for parallel download and pull the key into cache + try: + subprocess.call('axel') + self.use_axel = True + except OSError: + self.use_axel = False + + def _configure_connection( self ): + log.debug("Configuring AWS-S3 Connection") + aws_config = {'aws_access_key': self.access_key, + 'aws_secret_key': self.secret_key} + self.conn = CloudProviderFactory().create_provider(ProviderList.AWS, aws_config) + + def _parse_config_xml(self, config_xml): + try: + a_xml = config_xml.findall('auth')[0] + self.access_key = a_xml.get('access_key') + self.secret_key = a_xml.get('secret_key') + b_xml = config_xml.findall('bucket')[0] + self.bucket = b_xml.get('name') + self.use_rr = string_as_bool(b_xml.get('use_reduced_redundancy', "False")) + self.max_chunk_size = int(b_xml.get('max_chunk_size', 250)) + cn_xml = config_xml.findall('connection') + if not cn_xml: + cn_xml = {} + else: + cn_xml = cn_xml[0] + self.host = cn_xml.get('host', None) + self.port = int(cn_xml.get('port', 6000)) + self.multipart = string_as_bool(cn_xml.get('multipart', 'True')) + self.is_secure = string_as_bool(cn_xml.get('is_secure', 'True')) + self.conn_path = cn_xml.get('conn_path', '/') + c_xml = config_xml.findall('cache')[0] + self.cache_size = float(c_xml.get('size', -1)) + self.staging_path = c_xml.get('path', self.config.object_store_cache_path) + + for d_xml in config_xml.findall('extra_dir'): + self.extra_dirs[d_xml.get('type')] = d_xml.get('path') + + log.debug("Object cache dir: %s", self.staging_path) + log.debug(" job work dir: %s", self.extra_dirs['job_work']) + + except Exception: + # Toss it back up after logging, we can't continue loading at this point. + log.exception("Malformed ObjectStore Configuration XML -- unable to continue") + raise + + def __cache_monitor(self): + time.sleep(2) # Wait for things to load before starting the monitor + while self.running: + total_size = 0 + # Is this going to be too expensive of an operation to be done frequently? + file_list = [] + for dirpath, _, filenames in os.walk(self.staging_path): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + file_size = os.path.getsize(filepath) + total_size += file_size + # Get the time given file was last accessed + last_access_time = time.localtime(os.stat(filepath)[7]) + # Compose a tuple of the access time and the file path + file_tuple = last_access_time, filepath, file_size + file_list.append(file_tuple) + # Sort the file list (based on access time) + file_list.sort() + # Initiate cleaning once within 10% of the defined cache size? + cache_limit = self.cache_size * 0.9 + if total_size > cache_limit: + log.info("Initiating cache cleaning: current cache size: %s; clean until smaller than: %s", + convert_bytes(total_size), convert_bytes(cache_limit)) + # How much to delete? If simply deleting up to the cache-10% limit, + # is likely to be deleting frequently and may run the risk of hitting + # the limit - maybe delete additional #%? + # For now, delete enough to leave at least 10% of the total cache free + delete_this_much = total_size - cache_limit + self.__clean_cache(file_list, delete_this_much) + self.sleeper.sleep(30) # Test cache size every 30 seconds? + + def __clean_cache(self, file_list, delete_this_much): + """ Keep deleting files from the file_list until the size of the deleted + files is greater than the value in delete_this_much parameter. + + :type file_list: list + :param file_list: List of candidate files that can be deleted. This method + will start deleting files from the beginning of the list so the list + should be sorted accordingly. The list must contains 3-element tuples, + positioned as follows: position 0 holds file last accessed timestamp + (as time.struct_time), position 1 holds file path, and position 2 has + file size (e.g., (, /mnt/data/dataset_1.dat), 472394) + + :type delete_this_much: int + :param delete_this_much: Total size of files, in bytes, that should be deleted. + """ + # Keep deleting datasets from file_list until deleted_amount does not + # exceed delete_this_much; start deleting from the front of the file list, + # which assumes the oldest files come first on the list. + deleted_amount = 0 + for entry in enumerate(file_list): + if deleted_amount < delete_this_much: + deleted_amount += entry[2] + os.remove(entry[1]) + # Debugging code for printing deleted files' stats + # folder, file_name = os.path.split(f[1]) + # file_date = time.strftime("%m/%d/%y %H:%M:%S", f[0]) + # log.debug("%s. %-25s %s, size %s (deleted %s/%s)" \ + # % (i, file_name, convert_bytes(f[2]), file_date, \ + # convert_bytes(deleted_amount), convert_bytes(delete_this_much))) + else: + log.debug("Cache cleaning done. Total space freed: %s", convert_bytes(deleted_amount)) + return + + def _get_bucket(self, bucket_name): + """ Sometimes a handle to a bucket is not established right away so try + it a few times. Raise error if connection is not established. """ + for i in range(5): + try: + bucket = self.conn.object_store.get(bucket_name) + if bucket is None: + log.debug("Bucket not found, creating a bucket with handle '%s'", bucket_name) + bucket = self.conn.object_store.create(bucket_name) + log.debug("Using cloud object store with bucket '%s'", bucket.name) + return bucket + except S3ResponseError: + log.exception("Could not get bucket '%s', attempt %s/5", bucket_name, i + 1) + time.sleep(2) + # All the attempts have been exhausted and connection was not established, + # raise error + raise S3ResponseError + + def _fix_permissions(self, rel_path): + """ Set permissions on rel_path""" + for basedir, _, files in os.walk(rel_path): + umask_fix_perms(basedir, self.config.umask, 0o777, self.config.gid) + for filename in files: + path = os.path.join(basedir, filename) + # Ignore symlinks + if os.path.islink(path): + continue + umask_fix_perms(path, self.config.umask, 0o666, self.config.gid) + + def _construct_path(self, obj, base_dir=None, dir_only=None, extra_dir=None, extra_dir_at_root=False, alt_name=None, + obj_dir=False, **kwargs): + # extra_dir should never be constructed from provided data but just + # make sure there are no shenannigans afoot + if extra_dir and extra_dir != os.path.normpath(extra_dir): + log.warning('extra_dir is not normalized: %s', extra_dir) + raise ObjectInvalid("The requested object is invalid") + # ensure that any parent directory references in alt_name would not + # result in a path not contained in the directory path constructed here + if alt_name: + if not safe_relpath(alt_name): + log.warning('alt_name would locate path outside dir: %s', alt_name) + raise ObjectInvalid("The requested object is invalid") + # alt_name can contain parent directory references, but S3 will not + # follow them, so if they are valid we normalize them out + alt_name = os.path.normpath(alt_name) + rel_path = os.path.join(*directory_hash_id(obj.id)) + if extra_dir is not None: + if extra_dir_at_root: + rel_path = os.path.join(extra_dir, rel_path) + else: + rel_path = os.path.join(rel_path, extra_dir) + + # for JOB_WORK directory + if obj_dir: + rel_path = os.path.join(rel_path, str(obj.id)) + if base_dir: + base = self.extra_dirs.get(base_dir) + return os.path.join(base, rel_path) + + # S3 folders are marked by having trailing '/' so add it now + rel_path = '%s/' % rel_path + + if not dir_only: + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + return rel_path + + def _get_cache_path(self, rel_path): + return os.path.abspath(os.path.join(self.staging_path, rel_path)) + + def _get_transfer_progress(self): + return self.transfer_progress + + def _get_size_in_cloud(self, rel_path): + try: + obj = self.bucket.get(rel_path) + if obj: + return obj.size + except S3ResponseError: + log.exception("Could not get size of key '%s' from S3", rel_path) + return -1 + + def _key_exists(self, rel_path): + exists = False + try: + # A hackish way of testing if the rel_path is a folder vs a file + is_dir = rel_path[-1] == '/' + if is_dir: + keyresult = self.bucket.list(prefix=rel_path) + if len(keyresult) > 0: + exists = True + else: + exists = False + else: + exists = self.bucket.exists(rel_path) + except S3ResponseError: + log.exception("Trouble checking existence of S3 key '%s'", rel_path) + return False + if rel_path[0] == '/': + raise + return exists + + def _in_cache(self, rel_path): + """ Check if the given dataset is in the local cache and return True if so. """ + # log.debug("------ Checking cache for rel_path %s" % rel_path) + cache_path = self._get_cache_path(rel_path) + return os.path.exists(cache_path) + # TODO: Part of checking if a file is in cache should be to ensure the + # size of the cached file matches that on S3. Once the upload tool explicitly + # creates, this check sould be implemented- in the mean time, it's not + # looking likely to be implementable reliably. + # if os.path.exists(cache_path): + # # print "***1 %s exists" % cache_path + # if self._key_exists(rel_path): + # # print "***2 %s exists in S3" % rel_path + # # Make sure the size in cache is available in its entirety + # # print "File '%s' cache size: %s, S3 size: %s" % (cache_path, os.path.getsize(cache_path), self._get_size_in_cloud(rel_path)) + # if os.path.getsize(cache_path) == self._get_size_in_cloud(rel_path): + # # print "***2.1 %s exists in S3 and the size is the same as in cache (in_cache=True)" % rel_path + # exists = True + # else: + # # print "***2.2 %s exists but differs in size from cache (in_cache=False)" % cache_path + # exists = False + # else: + # # Although not perfect decision making, this most likely means + # # that the file is currently being uploaded + # # print "***3 %s found in cache but not in S3 (in_cache=True)" % cache_path + # exists = True + # else: + # return False + + def _pull_into_cache(self, rel_path): + # Ensure the cache directory structure exists (e.g., dataset_#_files/) + rel_path_dir = os.path.dirname(rel_path) + if not os.path.exists(self._get_cache_path(rel_path_dir)): + os.makedirs(self._get_cache_path(rel_path_dir)) + # Now pull in the file + file_ok = self._download(rel_path) + self._fix_permissions(self._get_cache_path(rel_path_dir)) + return file_ok + + def _transfer_cb(self, complete, total): + self.transfer_progress += 10 + + def _download(self, rel_path): + try: + log.debug("Pulling key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + key = self.bucket.get(rel_path) + # Test if cache is large enough to hold the new file + if self.cache_size > 0 and key.size > self.cache_size: + log.critical("File %s is larger (%s) than the cache size (%s). Cannot download.", + rel_path, key.size, self.cache_size) + return False + if self.use_axel: + log.debug("Parallel pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + ncores = multiprocessing.cpu_count() + url = key.generate_url(7200) + ret_code = subprocess.call("axel -a -n %s '%s'" % (ncores, url)) + if ret_code == 0: + return True + else: + log.debug("Pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) + self.transfer_progress = 0 # Reset transfer progress counter + with open(self._get_cache_path(rel_path), "w+") as downloaded_file_handle: + key.save_content(downloaded_file_handle) + return True + except S3ResponseError: + log.exception("Problem downloading key '%s' from S3 bucket '%s'", rel_path, self.bucket.name) + return False + + def _push_to_os(self, rel_path, source_file=None, from_string=None): + """ + Push the file pointed to by ``rel_path`` to the object store naming the key + ``rel_path``. If ``source_file`` is provided, push that file instead while + still using ``rel_path`` as the key name. + If ``from_string`` is provided, set contents of the file to the value of + the string. + """ + try: + source_file = source_file if source_file else self._get_cache_path(rel_path) + if os.path.exists(source_file): + if os.path.getsize(source_file) == 0 and self.bucket.exists(rel_path): + log.debug("Wanted to push file '%s' to S3 key '%s' but its size is 0; skipping.", source_file, + rel_path) + return True + # FIXME: don't need to differenciate between uploading from a string or file, + # because CloudBridge handles this internally. + if from_string: + if not self.bucket.get(rel_path): + created_obj = self.bucket.create_object(rel_path) + created_obj.upload(source_file) + else: + self.bucket.get(rel_path).upload(source_file) + log.debug("Pushed data from string '%s' to key '%s'", from_string, rel_path) + else: + start_time = datetime.now() + log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, + os.path.getsize(source_file), rel_path) + self.transfer_progress = 0 # Reset transfer progress counter + if not self.bucket.get(rel_path): + created_obj = self.bucket.create_object(rel_path) + created_obj.upload(source_file) + else: + self.bucket.get(rel_path).upload(source_file) + end_time = datetime.now() + log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", + source_file, rel_path, os.path.getsize(source_file), end_time - start_time) + return True + else: + log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", + rel_path, source_file) + except S3ResponseError: + log.exception("Trouble pushing S3 key '%s' from file '%s'", rel_path, source_file) + return False + + def file_ready(self, obj, **kwargs): + """ + A helper method that checks if a file corresponding to a dataset is + ready and available to be used. Return ``True`` if so, ``False`` otherwise. + """ + rel_path = self._construct_path(obj, **kwargs) + # Make sure the size in cache is available in its entirety + if self._in_cache(rel_path): + if os.path.getsize(self._get_cache_path(rel_path)) == self._get_size_in_cloud(rel_path): + return True + log.debug("Waiting for dataset %s to transfer from OS: %s/%s", rel_path, + os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_cloud(rel_path)) + return False + + def exists(self, obj, **kwargs): + in_cache = False + rel_path = self._construct_path(obj, **kwargs) + + # Check cache + if self._in_cache(rel_path): + in_cache = True + # Check cloud + in_cloud = self._key_exists(rel_path) + # log.debug("~~~~~~ File '%s' exists in cache: %s; in s3: %s" % (rel_path, in_cache, in_s3)) + # dir_only does not get synced so shortcut the decision + dir_only = kwargs.get('dir_only', False) + base_dir = kwargs.get('base_dir', None) + if dir_only: + if in_cache or in_cloud: + return True + # for JOB_WORK directory + elif base_dir: + if not os.path.exists(rel_path): + os.makedirs(rel_path) + return True + else: + return False + + # TODO: Sync should probably not be done here. Add this to an async upload stack? + if in_cache and not in_cloud: + self._push_to_os(rel_path, source_file=self._get_cache_path(rel_path)) + return True + elif in_cloud: + return True + else: + return False + + def create(self, obj, **kwargs): + if not self.exists(obj, **kwargs): + + # Pull out locally used fields + extra_dir = kwargs.get('extra_dir', None) + extra_dir_at_root = kwargs.get('extra_dir_at_root', False) + dir_only = kwargs.get('dir_only', False) + alt_name = kwargs.get('alt_name', None) + + # Construct hashed path + rel_path = os.path.join(*directory_hash_id(obj.id)) + + # Optionally append extra_dir + if extra_dir is not None: + if extra_dir_at_root: + rel_path = os.path.join(extra_dir, rel_path) + else: + rel_path = os.path.join(rel_path, extra_dir) + + # Create given directory in cache + cache_dir = os.path.join(self.staging_path, rel_path) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + # Although not really necessary to create S3 folders (because S3 has + # flat namespace), do so for consistency with the regular file system + # S3 folders are marked by having trailing '/' so add it now + # s3_dir = '%s/' % rel_path + # self._push_to_os(s3_dir, from_string='') + # If instructed, create the dataset in cache & in S3 + if not dir_only: + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + open(os.path.join(self.staging_path, rel_path), 'w').close() + self._push_to_os(rel_path, from_string='') + + def empty(self, obj, **kwargs): + if self.exists(obj, **kwargs): + return bool(self.size(obj, **kwargs) > 0) + else: + raise ObjectNotFound( 'objectstore.empty, object does not exist: %s, kwargs: %s' + % ( str( obj ), str( kwargs ) ) ) + + def size(self, obj, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + if self._in_cache(rel_path): + try: + return os.path.getsize(self._get_cache_path(rel_path)) + except OSError as ex: + log.info("Could not get size of file '%s' in local cache, will try cloud. Error: %s", rel_path, ex) + elif self.exists(obj, **kwargs): + return self._get_size_in_cloud(rel_path) + log.warning("Did not find dataset '%s', returning 0 for size", rel_path) + return 0 + + def delete(self, obj, entire_dir=False, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + extra_dir = kwargs.get('extra_dir', None) + base_dir = kwargs.get('base_dir', None) + dir_only = kwargs.get('dir_only', False) + obj_dir = kwargs.get('obj_dir', False) + try: + # Remove temparory data in JOB_WORK directory + if base_dir and dir_only and obj_dir: + shutil.rmtree(os.path.abspath(rel_path)) + return True + + # For the case of extra_files, because we don't have a reference to + # individual files/keys we need to remove the entire directory structure + # with all the files in it. This is easy for the local file system, + # but requires iterating through each individual key in S3 and deleing it. + if entire_dir and extra_dir: + shutil.rmtree(self._get_cache_path(rel_path)) + results = self.bucket.list(prefix=rel_path) + for key in results: + log.debug("Deleting key %s", key.name) + key.delete() + return True + else: + # Delete from cache first + os.unlink(self._get_cache_path(rel_path)) + # Delete from S3 as well + if self._key_exists(rel_path): + key = self.bucket.get(rel_path) + log.debug("Deleting key %s", key.name) + key.delete() + return True + except S3ResponseError: + log.exception("Could not delete key '%s' from cloud", rel_path) + except OSError: + log.exception('%s delete error', self.get_filename(obj, **kwargs)) + return False + + def get_data(self, obj, start=0, count=-1, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + # Check cache first and get file if not there + if not self._in_cache(rel_path): + self._pull_into_cache(rel_path) + # Read the file content from cache + data_file = open(self._get_cache_path(rel_path), 'r') + data_file.seek(start) + content = data_file.read(count) + data_file.close() + return content + + def get_filename(self, obj, **kwargs): + base_dir = kwargs.get('base_dir', None) + dir_only = kwargs.get('dir_only', False) + obj_dir = kwargs.get('obj_dir', False) + rel_path = self._construct_path(obj, **kwargs) + + # for JOB_WORK directory + if base_dir and dir_only and obj_dir: + return os.path.abspath(rel_path) + + cache_path = self._get_cache_path(rel_path) + # S3 does not recognize directories as files so cannot check if those exist. + # So, if checking dir only, ensure given dir exists in cache and return + # the expected cache path. + # dir_only = kwargs.get('dir_only', False) + # if dir_only: + # if not os.path.exists(cache_path): + # os.makedirs(cache_path) + # return cache_path + # Check if the file exists in the cache first + if self._in_cache(rel_path): + return cache_path + # Check if the file exists in persistent storage and, if it does, pull it into cache + elif self.exists(obj, **kwargs): + if dir_only: # Directories do not get pulled into cache + return cache_path + else: + if self._pull_into_cache(rel_path): + return cache_path + # For the case of retrieving a directory only, return the expected path + # even if it does not exist. + # if dir_only: + # return cache_path + raise ObjectNotFound( 'objectstore.get_filename, no cache_path: %s, kwargs: %s' + % ( str( obj ), str( kwargs ) ) ) + # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path + + def update_from_file(self, obj, file_name=None, create=False, **kwargs): + if create: + self.create(obj, **kwargs) + if self.exists(obj, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + # Chose whether to use the dataset file itself or an alternate file + if file_name: + source_file = os.path.abspath(file_name) + # Copy into cache + cache_file = self._get_cache_path(rel_path) + try: + if source_file != cache_file: + # FIXME? Should this be a `move`? + shutil.copy2(source_file, cache_file) + self._fix_permissions(cache_file) + except OSError: + log.exception("Trouble copying source file '%s' to cache '%s'", source_file, cache_file) + else: + source_file = self._get_cache_path(rel_path) + # Update the file on cloud + self._push_to_os(rel_path, source_file) + else: + raise ObjectNotFound( 'objectstore.update_from_file, object does not exist: %s, kwargs: %s' + % ( str( obj ), str( kwargs ) ) ) + + def get_object_url(self, obj, **kwargs): + if self.exists(obj, **kwargs): + rel_path = self._construct_path(obj, **kwargs) + try: + key = self.bucket.get(rel_path) + return key.generate_url(expires_in=86400) # 24hrs + except S3ResponseError: + log.exception("Trouble generating URL for dataset '%s'", rel_path) + return None + + def get_store_usage_percent(self): + return 0.0 From e4c00dd695bf23a506ffeec15f98a6f6d1a05312 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 25 Jul 2017 18:42:57 -0700 Subject: [PATCH 004/208] - Updated `Cloud` to account for changes in `CloudBridge` interface. - Added `Cloud` to objectstore import. --- lib/galaxy/objectstore/__init__.py | 3 +++ lib/galaxy/objectstore/cloud.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 30fc086c3cc..50b4a7937bb 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -722,6 +722,9 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None): elif store == 's3': from .s3 import S3ObjectStore return S3ObjectStore(config=config, config_xml=config_xml) + elif store == 'cloud': + from .cloud import Cloud + return Cloud(config=config, config_xml=config_xml) elif store == 'swift': from .s3 import SwiftObjectStore return SwiftObjectStore(config=config, config_xml=config_xml) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 809e92ad95a..06136f89385 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -267,7 +267,7 @@ class Cloud( ObjectStore ): else: exists = False else: - exists = self.bucket.exists(rel_path) + exists = True if self.bucket.get(rel_path) is not None else False except S3ResponseError: log.exception("Trouble checking existence of S3 key '%s'", rel_path) return False @@ -354,7 +354,7 @@ class Cloud( ObjectStore ): try: source_file = source_file if source_file else self._get_cache_path(rel_path) if os.path.exists(source_file): - if os.path.getsize(source_file) == 0 and self.bucket.exists(rel_path): + if os.path.getsize(source_file) == 0 and (self.bucket.get(rel_path) is not None): log.debug("Wanted to push file '%s' to S3 key '%s' but its size is 0; skipping.", source_file, rel_path) return True From ab34fc9bd92041065dd8f140194c56e0f7b2954f Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 26 Jul 2017 15:24:08 -0700 Subject: [PATCH 005/208] Updated `Babel` and `requests` packages version to meet the minimum requirements of `cloudbridge` package. --- lib/galaxy/dependencies/pinned-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 5cb2c8a1d9c..c5111a5db92 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -22,7 +22,7 @@ WebOb==1.4.1 WebHelpers==1.3 Mako==1.0.2 pytz==2015.4 -Babel==2.1.1 +Babel==2.4.0 Beaker==1.7.0 dictobj==0.3.1 nose==1.3.7 @@ -39,7 +39,7 @@ Markdown==2.6.3 # BioBlend and dependencies bioblend==0.7.0 boto==2.38.0 -requests==2.8.1 +requests==2.10.0 requests-toolbelt==0.4.0 # kombu and dependencies From e8541a71a4fdd7f975b3e8d0c6b054bdc33480ab Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 26 Jul 2017 16:07:42 -0700 Subject: [PATCH 006/208] Reverted the changes on package versions, as these changes are requests via a different PR. --- lib/galaxy/dependencies/pinned-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index c5111a5db92..5cb2c8a1d9c 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -22,7 +22,7 @@ WebOb==1.4.1 WebHelpers==1.3 Mako==1.0.2 pytz==2015.4 -Babel==2.4.0 +Babel==2.1.1 Beaker==1.7.0 dictobj==0.3.1 nose==1.3.7 @@ -39,7 +39,7 @@ Markdown==2.6.3 # BioBlend and dependencies bioblend==0.7.0 boto==2.38.0 -requests==2.10.0 +requests==2.8.1 requests-toolbelt==0.4.0 # kombu and dependencies From 7ad7d881046ddd20276998d6e1fca495d1ac9f50 Mon Sep 17 00:00:00 2001 From: vahid Date: Fri, 28 Jul 2017 09:26:35 -0700 Subject: [PATCH 007/208] Changed boto import style. --- lib/galaxy/objectstore/cloud.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 06136f89385..d72ae0d589b 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -25,10 +25,6 @@ from ..objectstore import convert_bytes, ObjectStore from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList try: - # Imports are done this way to allow objectstore code to be used outside of Galaxy. - import boto - - from boto.exception import S3ResponseError from boto.s3.key import Key from boto.s3.connection import S3Connection except ImportError: From 9c57adb90481b871cc6f98f58227caca4e414467 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 2 Aug 2017 10:05:14 -0700 Subject: [PATCH 008/208] Updated CloudBridge to its current latest version. --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 565ce7dd23f..466b2e9ab2e 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -13,7 +13,7 @@ graphitesend azure-storage==0.32.0 # PyRods not in PyPI python-ldap==2.4.27 -cloudbridge==0.3.1 +cloudbridge==0.3.2 # Synnefo / Pithos+ object store client kamaki From 2589c745ae7f88ada7a08265f1238ae00274425e Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 2 Aug 2017 11:29:25 -0700 Subject: [PATCH 009/208] Commented on boto import, and removed a `use_reduced_redundancy` config. --- lib/galaxy/objectstore/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index d72ae0d589b..7254f076e5c 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -24,7 +24,9 @@ from galaxy.util.sleeper import Sleeper from ..objectstore import convert_bytes, ObjectStore from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList +# boto is only used to handle exceptions; it will be removed once CloudBridge wraps and throws proper exceptions. try: + # Imports are done this way to allow objectstore code to be used outside of Galaxy. from boto.s3.key import Key from boto.s3.connection import S3Connection except ImportError: @@ -79,7 +81,6 @@ class Cloud( ObjectStore ): self.secret_key = a_xml.get('secret_key') b_xml = config_xml.findall('bucket')[0] self.bucket = b_xml.get('name') - self.use_rr = string_as_bool(b_xml.get('use_reduced_redundancy', "False")) self.max_chunk_size = int(b_xml.get('max_chunk_size', 250)) cn_xml = config_xml.findall('connection') if not cn_xml: From 89be432770d3cf9442f9c003ead8c6ea6a16f395 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 2 Aug 2017 11:54:13 -0700 Subject: [PATCH 010/208] Removed boto import, and replaced all S3-specific exception catches with a generic exception--a temporary solution till CloudBridge wraps the exceptions properly. --- lib/galaxy/objectstore/cloud.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 7254f076e5c..def78bbe62d 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -24,14 +24,6 @@ from galaxy.util.sleeper import Sleeper from ..objectstore import convert_bytes, ObjectStore from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList -# boto is only used to handle exceptions; it will be removed once CloudBridge wraps and throws proper exceptions. -try: - # Imports are done this way to allow objectstore code to be used outside of Galaxy. - from boto.s3.key import Key - from boto.s3.connection import S3Connection -except ImportError: - boto = None - NO_BOTO_ERROR_MESSAGE = ("Cloud object store is configured, but no boto dependency available." "Please install and properly configure boto or modify object store configuration.") @@ -182,12 +174,12 @@ class Cloud( ObjectStore ): bucket = self.conn.object_store.create(bucket_name) log.debug("Using cloud object store with bucket '%s'", bucket.name) return bucket - except S3ResponseError: + except Exception: log.exception("Could not get bucket '%s', attempt %s/5", bucket_name, i + 1) time.sleep(2) # All the attempts have been exhausted and connection was not established, # raise error - raise S3ResponseError + raise Exception def _fix_permissions(self, rel_path): """ Set permissions on rel_path""" @@ -248,7 +240,7 @@ class Cloud( ObjectStore ): obj = self.bucket.get(rel_path) if obj: return obj.size - except S3ResponseError: + except Exception: log.exception("Could not get size of key '%s' from S3", rel_path) return -1 @@ -265,7 +257,7 @@ class Cloud( ObjectStore ): exists = False else: exists = True if self.bucket.get(rel_path) is not None else False - except S3ResponseError: + except Exception: log.exception("Trouble checking existence of S3 key '%s'", rel_path) return False if rel_path[0] == '/': @@ -336,7 +328,7 @@ class Cloud( ObjectStore ): with open(self._get_cache_path(rel_path), "w+") as downloaded_file_handle: key.save_content(downloaded_file_handle) return True - except S3ResponseError: + except Exception: log.exception("Problem downloading key '%s' from S3 bucket '%s'", rel_path, self.bucket.name) return False @@ -381,7 +373,7 @@ class Cloud( ObjectStore ): else: log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", rel_path, source_file) - except S3ResponseError: + except Exception: log.exception("Trouble pushing S3 key '%s' from file '%s'", rel_path, source_file) return False @@ -518,7 +510,7 @@ class Cloud( ObjectStore ): log.debug("Deleting key %s", key.name) key.delete() return True - except S3ResponseError: + except Exception: log.exception("Could not delete key '%s' from cloud", rel_path) except OSError: log.exception('%s delete error', self.get_filename(obj, **kwargs)) @@ -604,7 +596,7 @@ class Cloud( ObjectStore ): try: key = self.bucket.get(rel_path) return key.generate_url(expires_in=86400) # 24hrs - except S3ResponseError: + except Exception: log.exception("Trouble generating URL for dataset '%s'", rel_path) return None From fafb019f4be3b33ad9cab4bfd2ded6788b419592 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 2 Aug 2017 12:05:45 -0700 Subject: [PATCH 011/208] Consolidated two upload methods, i.e., upload from file and string, because CloudBridge internally maps to appropriate functions depending on the input type. --- lib/galaxy/objectstore/cloud.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index def78bbe62d..dac5dd6baa6 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -347,28 +347,18 @@ class Cloud( ObjectStore ): log.debug("Wanted to push file '%s' to S3 key '%s' but its size is 0; skipping.", source_file, rel_path) return True - # FIXME: don't need to differenciate between uploading from a string or file, - # because CloudBridge handles this internally. - if from_string: - if not self.bucket.get(rel_path): - created_obj = self.bucket.create_object(rel_path) - created_obj.upload(source_file) - else: - self.bucket.get(rel_path).upload(source_file) - log.debug("Pushed data from string '%s' to key '%s'", from_string, rel_path) + start_time = datetime.now() + log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, + os.path.getsize(source_file), rel_path) + self.transfer_progress = 0 # Reset transfer progress counter + if not self.bucket.get(rel_path): + created_obj = self.bucket.create_object(rel_path) + created_obj.upload(source_file) else: - start_time = datetime.now() - log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, - os.path.getsize(source_file), rel_path) - self.transfer_progress = 0 # Reset transfer progress counter - if not self.bucket.get(rel_path): - created_obj = self.bucket.create_object(rel_path) - created_obj.upload(source_file) - else: - self.bucket.get(rel_path).upload(source_file) - end_time = datetime.now() - log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", - source_file, rel_path, os.path.getsize(source_file), end_time - start_time) + self.bucket.get(rel_path).upload(source_file) + end_time = datetime.now() + log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", + source_file, rel_path, os.path.getsize(source_file), end_time - start_time) return True else: log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", From fb25495d191c2172ce22496900db2310d6fc9b69 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 2 Aug 2017 12:11:26 -0700 Subject: [PATCH 012/208] Removed a S3-related comment, and swapped an if/else condition. --- lib/galaxy/objectstore/cloud.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index dac5dd6baa6..432f1b01dd9 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -351,11 +351,11 @@ class Cloud( ObjectStore ): log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, os.path.getsize(source_file), rel_path) self.transfer_progress = 0 # Reset transfer progress counter - if not self.bucket.get(rel_path): + if self.bucket.get(rel_path): + self.bucket.get(rel_path).upload(source_file) + else: created_obj = self.bucket.create_object(rel_path) created_obj.upload(source_file) - else: - self.bucket.get(rel_path).upload(source_file) end_time = datetime.now() log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", source_file, rel_path, os.path.getsize(source_file), end_time - start_time) @@ -438,12 +438,6 @@ class Cloud( ObjectStore ): if not os.path.exists(cache_dir): os.makedirs(cache_dir) - # Although not really necessary to create S3 folders (because S3 has - # flat namespace), do so for consistency with the regular file system - # S3 folders are marked by having trailing '/' so add it now - # s3_dir = '%s/' % rel_path - # self._push_to_os(s3_dir, from_string='') - # If instructed, create the dataset in cache & in S3 if not dir_only: rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) open(os.path.join(self.staging_path, rel_path), 'w').close() From 7727c0bf627af5905c6e029417602389238f120e Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 2 Aug 2017 12:33:53 -0700 Subject: [PATCH 013/208] Removed a comment section. --- lib/galaxy/objectstore/cloud.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 432f1b01dd9..f6d0b3c7217 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -269,29 +269,6 @@ class Cloud( ObjectStore ): # log.debug("------ Checking cache for rel_path %s" % rel_path) cache_path = self._get_cache_path(rel_path) return os.path.exists(cache_path) - # TODO: Part of checking if a file is in cache should be to ensure the - # size of the cached file matches that on S3. Once the upload tool explicitly - # creates, this check sould be implemented- in the mean time, it's not - # looking likely to be implementable reliably. - # if os.path.exists(cache_path): - # # print "***1 %s exists" % cache_path - # if self._key_exists(rel_path): - # # print "***2 %s exists in S3" % rel_path - # # Make sure the size in cache is available in its entirety - # # print "File '%s' cache size: %s, S3 size: %s" % (cache_path, os.path.getsize(cache_path), self._get_size_in_cloud(rel_path)) - # if os.path.getsize(cache_path) == self._get_size_in_cloud(rel_path): - # # print "***2.1 %s exists in S3 and the size is the same as in cache (in_cache=True)" % rel_path - # exists = True - # else: - # # print "***2.2 %s exists but differs in size from cache (in_cache=False)" % cache_path - # exists = False - # else: - # # Although not perfect decision making, this most likely means - # # that the file is currently being uploaded - # # print "***3 %s found in cache but not in S3 (in_cache=True)" % cache_path - # exists = True - # else: - # return False def _pull_into_cache(self, rel_path): # Ensure the cache directory structure exists (e.g., dataset_#_files/) From 37efbd03ce4215086d29e8f4e9fd977b8e1299e0 Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 6 Aug 2017 22:48:42 -0700 Subject: [PATCH 014/208] Changed indentation of two lines. --- lib/galaxy/objectstore/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index f6d0b3c7217..18f5da4d507 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -326,7 +326,7 @@ class Cloud( ObjectStore ): return True start_time = datetime.now() log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, - os.path.getsize(source_file), rel_path) + os.path.getsize(source_file), rel_path) self.transfer_progress = 0 # Reset transfer progress counter if self.bucket.get(rel_path): self.bucket.get(rel_path).upload(source_file) @@ -335,7 +335,7 @@ class Cloud( ObjectStore ): created_obj.upload(source_file) end_time = datetime.now() log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", - source_file, rel_path, os.path.getsize(source_file), end_time - start_time) + source_file, rel_path, os.path.getsize(source_file), end_time - start_time) return True else: log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", From 9a5016fdcf4f36d5c20991f44d4b02fe7904ef66 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 17 Aug 2017 01:55:52 -0700 Subject: [PATCH 015/208] Update CloudBridge to its current latest Should use the wheel created at [this PR](https://github.com/galaxyproject/starforge/pull/139). --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 466b2e9ab2e..8aaeb1746ea 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -13,7 +13,7 @@ graphitesend azure-storage==0.32.0 # PyRods not in PyPI python-ldap==2.4.27 -cloudbridge==0.3.2 +cloudbridge==0.3.3 # Synnefo / Pithos+ object store client kamaki From 4e17c8e3b790d2d21bcf12c3d4a855b1226d810d Mon Sep 17 00:00:00 2001 From: vahid Date: Thu, 17 Aug 2017 19:05:25 -0700 Subject: [PATCH 016/208] Add Cloud configuration to object_store_conf.xml.sample --- config/object_store_conf.xml.sample | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/object_store_conf.xml.sample b/config/object_store_conf.xml.sample index 655e7271fbe..59170e046ca 100644 --- a/config/object_store_conf.xml.sample +++ b/config/object_store_conf.xml.sample @@ -52,6 +52,16 @@ --> + + From a8784369e4b243856b9582f9417039bf0c6bc83a Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 23 Aug 2017 12:41:15 -0700 Subject: [PATCH 017/208] Removed white spaces. --- lib/galaxy/objectstore/cloud.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 18f5da4d507..7ffdb32274e 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -27,21 +27,21 @@ from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList NO_BOTO_ERROR_MESSAGE = ("Cloud object store is configured, but no boto dependency available." "Please install and properly configure boto or modify object store configuration.") -log = logging.getLogger( __name__ ) +log = logging.getLogger(__name__) logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy -class Cloud( ObjectStore ): +class Cloud(ObjectStore): """ Object store that stores objects as items in an cloud storage. A local cache exists that is used as an intermediate location for files between Galaxy and the cloud storage. """ - def __init__( self, config, config_xml ): - super( Cloud, self ).__init__( config ) + def __init__(self, config, config_xml): + super(Cloud, self).__init__(config) self.staging_path = self.config.file_path self.transfer_progress = 0 - self._parse_config_xml( config_xml ) + self._parse_config_xml(config_xml) self._configure_connection() self.bucket = self._get_bucket(self.bucket) # Clean cache only if value is set in galaxy.ini @@ -60,7 +60,7 @@ class Cloud( ObjectStore ): except OSError: self.use_axel = False - def _configure_connection( self ): + def _configure_connection(self): log.debug("Configuring AWS-S3 Connection") aws_config = {'aws_access_key': self.access_key, 'aws_secret_key': self.secret_key} @@ -424,8 +424,8 @@ class Cloud( ObjectStore ): if self.exists(obj, **kwargs): return bool(self.size(obj, **kwargs) > 0) else: - raise ObjectNotFound( 'objectstore.empty, object does not exist: %s, kwargs: %s' - % ( str( obj ), str( kwargs ) ) ) + raise ObjectNotFound('objectstore.empty, object does not exist: %s, kwargs: %s' + % (str(obj), str(kwargs))) def size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) @@ -522,8 +522,8 @@ class Cloud( ObjectStore ): # even if it does not exist. # if dir_only: # return cache_path - raise ObjectNotFound( 'objectstore.get_filename, no cache_path: %s, kwargs: %s' - % ( str( obj ), str( kwargs ) ) ) + raise ObjectNotFound('objectstore.get_filename, no cache_path: %s, kwargs: %s' + % (str(obj), str(kwargs))) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path def update_from_file(self, obj, file_name=None, create=False, **kwargs): @@ -548,8 +548,8 @@ class Cloud( ObjectStore ): # Update the file on cloud self._push_to_os(rel_path, source_file) else: - raise ObjectNotFound( 'objectstore.update_from_file, object does not exist: %s, kwargs: %s' - % ( str( obj ), str( kwargs ) ) ) + raise ObjectNotFound('objectstore.update_from_file, object does not exist: %s, kwargs: %s' + % (str(obj), str(kwargs))) def get_object_url(self, obj, **kwargs): if self.exists(obj, **kwargs): From eaa5931d195d4710d939e6f8d53bfe2fed38ba2f Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 23 Aug 2017 13:29:43 -0700 Subject: [PATCH 018/208] Updated requirements: sqlalchemy-migrate and pbr. --- lib/galaxy/dependencies/pinned-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 9c8eab78bd0..2801b8905ec 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -58,11 +58,11 @@ psutil==4.1.0 pulsar-galaxy-lib==0.7.0.dev5 # sqlalchemy-migrate and dependencies -sqlalchemy-migrate==0.10.0 +sqlalchemy-migrate==0.11.0 decorator==4.0.2 Tempita==0.5.3dev sqlparse==0.1.16 -pbr==1.8.0 +pbr==2.0.0 # svgwrite and dependencies svgwrite==1.1.6 From 90c444d9a136400af608cb7fd7f8e8d59de3ba90 Mon Sep 17 00:00:00 2001 From: vahid Date: Wed, 23 Aug 2017 18:05:00 -0700 Subject: [PATCH 019/208] Removed a white space between parenthesis and arguments --- lib/galaxy/dependencies/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index eee90ce5ddb..4bceea79620 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -109,7 +109,7 @@ class ConditionalDependencies(object): def check_azure_storage(self): return 'azure_blob' in self.object_stores - def check_cloudbridge( self ): + def check_cloudbridge(self): return 'cloud' in self.object_stores def check_kamaki(self): From f6586a8e6d861c8c6803f96340641abf6470c2fc Mon Sep 17 00:00:00 2001 From: vahid Date: Thu, 24 Aug 2017 11:43:21 -0700 Subject: [PATCH 020/208] Surrounded CloudBridge import in a try-catch block. --- lib/galaxy/objectstore/cloud.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 7ffdb32274e..94510bd000a 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -20,16 +20,19 @@ from galaxy.util import ( umask_fix_perms, ) from galaxy.util.sleeper import Sleeper - from ..objectstore import convert_bytes, ObjectStore -from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList - -NO_BOTO_ERROR_MESSAGE = ("Cloud object store is configured, but no boto dependency available." - "Please install and properly configure boto or modify object store configuration.") log = logging.getLogger(__name__) logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy +try: + from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList +except ImportError: + log.error("Could not import CloudBridge.") + +NO_BOTO_ERROR_MESSAGE = ("Cloud object store is configured, but no boto dependency available." + "Please install and properly configure boto or modify object store configuration.") + class Cloud(ObjectStore): """ From 26cbf04ed0f30f72c994af0ffab54eab7dd8ac0b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 5 Sep 2017 09:36:39 -0400 Subject: [PATCH 021/208] Rework conditional dependency handling in galaxy.objectstore.cloud. This file may be loaded (e.g. for testing) but not used, so we shouldn't log a generic error about cloudbridge being unavailable, if someone attempts to actually use the object store and it isn't available then raise an informative exception. --- lib/galaxy/objectstore/cloud.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 94510bd000a..316b9493d4a 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -22,16 +22,20 @@ from galaxy.util import ( from galaxy.util.sleeper import Sleeper from ..objectstore import convert_bytes, ObjectStore -log = logging.getLogger(__name__) -logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy - try: from cloudbridge.cloud.factory import CloudProviderFactory, ProviderList except ImportError: - log.error("Could not import CloudBridge.") + CloudProviderFactory = None + ProviderList = None -NO_BOTO_ERROR_MESSAGE = ("Cloud object store is configured, but no boto dependency available." - "Please install and properly configure boto or modify object store configuration.") +log = logging.getLogger(__name__) + +logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy + +NO_CLOUDBRIDGE_ERROR_MESSAGE = ( + "ObjectStore configured, but no cloudbridge dependency available." + "Please install cloudbridge or modify Object Store configuration." +) class Cloud(ObjectStore): @@ -42,6 +46,8 @@ class Cloud(ObjectStore): """ def __init__(self, config, config_xml): super(Cloud, self).__init__(config) + if CloudProviderFactory is None: + raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) self.staging_path = self.config.file_path self.transfer_progress = 0 self._parse_config_xml(config_xml) From 6d65e902b12b4fca9af613085b5bc828c8a6d2fc Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 5 Sep 2017 10:57:36 -0700 Subject: [PATCH 022/208] Updated an error message. --- lib/galaxy/objectstore/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 316b9493d4a..191bcf8d4bd 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -33,8 +33,8 @@ log = logging.getLogger(__name__) logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy NO_CLOUDBRIDGE_ERROR_MESSAGE = ( - "ObjectStore configured, but no cloudbridge dependency available." - "Please install cloudbridge or modify Object Store configuration." + "Cloud ObjectStore is configured, but no CloudBridge dependency available." + "Please install CloudBridge or modify ObjectStore configuration." ) From cfce6c221c3f453e75542d53262f53c7e3411f86 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 5 Sep 2017 10:58:23 -0700 Subject: [PATCH 023/208] Removed `boto` logging level set. --- lib/galaxy/objectstore/cloud.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 191bcf8d4bd..7edef0a0144 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -30,8 +30,6 @@ except ImportError: log = logging.getLogger(__name__) -logging.getLogger('boto').setLevel(logging.INFO) # Otherwise boto is quite noisy - NO_CLOUDBRIDGE_ERROR_MESSAGE = ( "Cloud ObjectStore is configured, but no CloudBridge dependency available." "Please install CloudBridge or modify ObjectStore configuration." From b55583701cbf33f52db88f147dda6b04499243d7 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 5 Sep 2017 11:04:26 -0700 Subject: [PATCH 024/208] Removed retries for getting a bucket. --- lib/galaxy/objectstore/cloud.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 7edef0a0144..7cba1715b85 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -171,21 +171,17 @@ class Cloud(ObjectStore): return def _get_bucket(self, bucket_name): - """ Sometimes a handle to a bucket is not established right away so try - it a few times. Raise error if connection is not established. """ - for i in range(5): - try: - bucket = self.conn.object_store.get(bucket_name) - if bucket is None: - log.debug("Bucket not found, creating a bucket with handle '%s'", bucket_name) - bucket = self.conn.object_store.create(bucket_name) - log.debug("Using cloud object store with bucket '%s'", bucket.name) - return bucket - except Exception: - log.exception("Could not get bucket '%s', attempt %s/5", bucket_name, i + 1) - time.sleep(2) - # All the attempts have been exhausted and connection was not established, - # raise error + try: + bucket = self.conn.object_store.get(bucket_name) + if bucket is None: + log.debug("Bucket not found, creating a bucket with handle '%s'", bucket_name) + bucket = self.conn.object_store.create(bucket_name) + log.debug("Using cloud ObjectStore with bucket '%s'", bucket.name) + return bucket + except Exception: + # These two generic exceptions will be replaced by specific exceptions + # once proper exceptions are exposed by CloudBridge. + log.exception("Could not get bucket '%s'.", bucket_name) raise Exception def _fix_permissions(self, rel_path): From 5977056715a5d2e12b638012b6fc99d885ea034a Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 11 Sep 2017 22:20:22 -0700 Subject: [PATCH 025/208] Resolved a bug with uploading a dataset using Cloud (not differentiating between uploading from a file vs. string). --- lib/galaxy/objectstore/cloud.py | 34 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 7cba1715b85..01ee1e9a006 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -327,18 +327,30 @@ class Cloud(ObjectStore): log.debug("Wanted to push file '%s' to S3 key '%s' but its size is 0; skipping.", source_file, rel_path) return True - start_time = datetime.now() - log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, - os.path.getsize(source_file), rel_path) - self.transfer_progress = 0 # Reset transfer progress counter - if self.bucket.get(rel_path): - self.bucket.get(rel_path).upload(source_file) + if from_string: + if not self.bucket.get(rel_path): + created_obj = self.bucket.create_object(rel_path) + created_obj.upload(source_file) + else: + self.bucket.get(rel_path).upload(source_file) + log.debug("Pushed data from string '%s' to key '%s'", from_string, rel_path) else: - created_obj = self.bucket.create_object(rel_path) - created_obj.upload(source_file) - end_time = datetime.now() - log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", - source_file, rel_path, os.path.getsize(source_file), end_time - start_time) + start_time = datetime.now() + log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, + os.path.getsize(source_file), rel_path) + mb_size = os.path.getsize(source_file) / 1e6 + self.transfer_progress = 0 # Reset transfer progress counter + if not self.bucket.get(rel_path): + created_obj = self.bucket.create_object(rel_path) + created_obj.upload_from_file(source_file) + else: + self.bucket.get(rel_path).upload_from_file(source_file) + # else: + # multipart_upload(self.s3server, bucket, bucket.get(rel_path).name, source_file, mb_size) + + end_time = datetime.now() + log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", + source_file, rel_path, os.path.getsize(source_file), end_time - start_time) return True else: log.error("Tried updating key '%s' from source file '%s', but source file does not exist.", From 45ac767a4eb6ab9e15a1e46a1e9e4ae70591e188 Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 11 Sep 2017 23:02:27 -0700 Subject: [PATCH 026/208] Removed an unused variable (i.e., mb_size) from Cloud --- lib/galaxy/objectstore/cloud.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 01ee1e9a006..2c5569dd07a 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -338,15 +338,12 @@ class Cloud(ObjectStore): start_time = datetime.now() log.debug("Pushing cache file '%s' of size %s bytes to key '%s'", source_file, os.path.getsize(source_file), rel_path) - mb_size = os.path.getsize(source_file) / 1e6 self.transfer_progress = 0 # Reset transfer progress counter if not self.bucket.get(rel_path): created_obj = self.bucket.create_object(rel_path) created_obj.upload_from_file(source_file) else: self.bucket.get(rel_path).upload_from_file(source_file) - # else: - # multipart_upload(self.s3server, bucket, bucket.get(rel_path).name, source_file, mb_size) end_time = datetime.now() log.debug("Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)", From a54f4892fa1defe5e406e10a658305872f479be3 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 14:34:19 -0500 Subject: [PATCH 027/208] Remove generic formbuilder controller redirect --- lib/galaxy/web/framework/webapp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index 831474246a1..37eb57365c7 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -33,7 +33,6 @@ from galaxy.util import ( from galaxy.util.sanitize_html import sanitize_html from galaxy.web.framework import ( base, - formbuilder, helpers, url_for ) @@ -103,8 +102,6 @@ class WebApplication(base.WebApplication): return trans.show_message(sanitize_html(e.err_msg), e.type) def make_body_iterable(self, trans, body): - if isinstance(body, formbuilder.FormBuilder): - body = trans.show_form(body) return base.WebApplication.make_body_iterable(self, trans, body) def transaction_chooser(self, environ, galaxy_app, session_cookie): From 6f2da1882a119db41adbfc766add198733522ea7 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 14:45:58 -0500 Subject: [PATCH 028/208] Remove formbuilder code file --- lib/galaxy/web/framework/formbuilder.py | 91 ------------------------- 1 file changed, 91 deletions(-) delete mode 100644 lib/galaxy/web/framework/formbuilder.py diff --git a/lib/galaxy/web/framework/formbuilder.py b/lib/galaxy/web/framework/formbuilder.py deleted file mode 100644 index 1603e3bf4a8..00000000000 --- a/lib/galaxy/web/framework/formbuilder.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging - -from galaxy.util import bunch - -log = logging.getLogger(__name__) - - -def form(*args, **kwargs): - return FormBuilder(*args, **kwargs) - - -class FormBuilder(object): - """ - Simple class describing an HTML form - """ - - def __init__(self, action="", title="", name="form", submit_text="submit", use_panels=False): - self.title = title - self.name = name - self.action = action - self.submit_text = submit_text - self.inputs = [] - self.use_panels = use_panels - - def add_input(self, type, name, label, value=None, error=None, help=None, use_label=True): - self.inputs.append(FormInput(type, label, name, value, error, help, use_label)) - return self - - def add_checkbox(self, name, label, value=None, error=None, help=None): - return self.add_input('checkbox', label, name, value, error, help) - - def add_text(self, name, label, value=None, error=None, help=None): - return self.add_input('text', label, name, value, error, help) - - def add_password(self, name, label, value=None, error=None, help=None): - return self.add_input('password', label, name, value, error, help) - - def add_select(self, name, label, value=None, options=[], error=None, help=None, use_label=True): - self.inputs.append(SelectInput(name, label, value=value, options=options, error=error, help=help, use_label=use_label)) - return self - - -class FormInput(object): - """ - Simple class describing a form input element - """ - - def __init__(self, type, name, label, value=None, error=None, help=None, use_label=True, extra_attributes={}, **kwargs): - self.type = type - self.name = name - self.label = label - self.value = value - self.error = error - self.help = help - self.use_label = use_label - self.extra_attributes = extra_attributes - - -class DatalistInput(FormInput): - """ Data list input """ - - def __init__(self, name, *args, **kwargs): - if 'extra_attributes' not in kwargs: - kwargs['extra_attributes'] = {} - kwargs['extra_attributes']['list'] = name - FormInput.__init__(self, None, name, *args, **kwargs) - self.options = kwargs.get('options', {}) - - def body_html(self): - options = "".join(["" % (key, value) for key, value in self.options.iteritems()]) - return """%s""" % (self.name, options) - - -class SelectInput(FormInput): - """ A select form input. """ - - def __init__(self, name, label, value=None, options=[], error=None, help=None, use_label=True): - FormInput.__init__(self, "select", name, label, value=value, error=error, help=help, use_label=use_label) - self.options = options - - -class FormData(object): - """ - Class for passing data about a form to a template, very rudimentary, could - be combined with the tool form handling to build something more general. - """ - - def __init__(self): - # TODO: galaxy's two Bunchs are defined differently. Is this right? - self.values = bunch.Bunch() - self.errors = bunch.Bunch() From 28f7bed5ffcc85069d234724c56a6a0850f1da2c Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 14:54:05 -0500 Subject: [PATCH 029/208] Remove unused field as html function from model --- lib/galaxy/model/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a3a48c5c264..750e5b0aad7 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4528,20 +4528,6 @@ class FormDefinition(object, Dictifiable): widgets.append(dict(label=field['label'], widget=field_widget, helptext=helptext)) return widgets - def field_as_html(self, field): - """Generates disabled html for a field""" - type = field['type'] - form_field = None - for field_type in self.supported_field_types: - if type == field_type.__name__: - # Name it AddressField, CheckboxField, etc. - form_field = field_type(type) - break - if form_field: - return form_field.get_html(disabled=True) - # Return None if unsupported field type - return None - class FormDefinitionCurrent(object): def __init__(self, form_definition=None): From 67aac2d9d9d4305f8064584366125579445cc476 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 14:55:01 -0500 Subject: [PATCH 030/208] Remove unused html helper for address fields --- lib/galaxy/model/__init__.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 750e5b0aad7..4760b4ce1f9 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4567,30 +4567,6 @@ class UserAddress(object): 'country' : sanitize_html(self.country), 'phone' : sanitize_html(self.phone)} - def get_html(self): - # This should probably be deprecated eventually. It should currently - # sanitize. - # TODO Find out where else uses this and replace with - # templates - html = '' - if self.name: - html = html + sanitize_html(self.name) - if self.institution: - html = html + '
' + sanitize_html(self.institution) - if self.address: - html = html + '
' + sanitize_html(self.address) - if self.city: - html = html + '
' + sanitize_html(self.city) - if self.state: - html = html + ' ' + sanitize_html(self.state) - if self.postal_code: - html = html + ' ' + sanitize_html(self.postal_code) - if self.country: - html = html + '
' + sanitize_html(self.country) - if self.phone: - html = html + '
' + 'phone: ' + sanitize_html(self.phone) - return html - class UserOpenID(object): def __init__(self, user=None, session=None, openid=None): From 372ae48a85ddf9cc5f450bba19e937e23a6a3c08 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:04:20 -0500 Subject: [PATCH 031/208] Remove html helpers from metadata parameter inputs --- lib/galaxy/model/metadata.py | 69 ------------------------------------ 1 file changed, 69 deletions(-) diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index ba937e61391..817a1a809e9 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -227,29 +227,6 @@ class MetadataParameter(object): other_values = other_values or {} return form_builder.TextField(self.spec.name, value=value) - def get_html(self, value, context=None, other_values=None, **kwd): - """ - The "context" is simply the metadata collection/bunch holding - this piece of metadata. This is passed in to allow for - metadata to validate against each other (note: this could turn - into a huge, recursive mess if not done with care). For - example, a column assignment should validate against the - number of columns in the dataset. - """ - context = context or {} - other_values = other_values or {} - - if self.spec.get("readonly"): - return value - if self.spec.get("optional"): - checked = False - if value: - checked = "true" - checkbox = form_builder.CheckboxField("is_" + self.spec.name, checked=checked) - return checkbox.get_html() + self.get_field(value=value, context=context, other_values=other_values, **kwd).get_html() - else: - return self.get_field(value=value, context=context, other_values=other_values, **kwd).get_html() - def to_string(self, value): return str(value) @@ -390,16 +367,6 @@ class SelectParameter(MetadataParameter): field.add_option(val, label, selected=False) return field - def get_html(self, value, context=None, other_values=None, values=None, **kwd): - context = context or {} - other_values = other_values or {} - - if self.spec.get("readonly"): - if value in [None, []]: - return str(self.spec.no_value) - return ", ".join(map(str, value)) - return MetadataParameter.get_html(self, value, context=context, other_values=other_values, values=values, **kwd) - def wrap(self, value, session): # do we really need this (wasteful)? - yes because we are not sure that # all existing selects have been stored previously as lists. Also this @@ -433,15 +400,6 @@ class DBKeyParameter(SelectParameter): pass return super(DBKeyParameter, self).get_field(value, context, other_values, values, **kwd) - def get_html(self, value=None, context=None, other_values=None, values=None, **kwd): - context = context or {} - other_values = other_values or {} - try: - values = kwd['trans'].app.genome_builds.get_genome_build_names(kwd['trans']) - except KeyError: - pass - return super(DBKeyParameter, self).get_html(value, context, other_values, values, **kwd) - class RangeParameter(SelectParameter): @@ -460,14 +418,6 @@ class RangeParameter(SelectParameter): values = list(zip(range(self.min, self.max, self.step), range(self.min, self.max, self.step))) return SelectParameter.get_field(self, value=value, context=context, other_values=other_values, values=values, **kwd) - def get_html(self, value, context=None, other_values=None, values=None, **kwd): - context = context or {} - other_values = other_values or {} - - if values is None: - values = list(zip(range(self.min, self.max, self.step), range(self.min, self.max, self.step))) - return SelectParameter.get_html(self, value, context=context, other_values=other_values, values=values, **kwd) - @classmethod def marshal(cls, value): value = SelectParameter.marshal(value) @@ -486,15 +436,6 @@ class ColumnParameter(RangeParameter): values = list(zip(column_range, column_range)) return RangeParameter.get_field(self, value=value, context=context, other_values=other_values, values=values, **kwd) - def get_html(self, value, context=None, other_values=None, values=None, **kwd): - context = context or {} - other_values = other_values or {} - - if values is None and context: - column_range = range(1, (context.columns or 0) + 1, 1) - values = list(zip(column_range, column_range)) - return RangeParameter.get_html(self, value, context=context, other_values=other_values, values=values, **kwd) - class ColumnTypesParameter(MetadataParameter): @@ -530,11 +471,6 @@ class PythonObjectParameter(MetadataParameter): other_values = other_values or {} return form_builder.TextField(self.spec.name, value=self._to_string(value)) - def get_html(self, value=None, context=None, other_values=None, **kwd): - context = context or {} - other_values = other_values or {} - return str(self) - @classmethod def marshal(cls, value): return value @@ -556,11 +492,6 @@ class FileParameter(MetadataParameter): other_values = other_values or {} return form_builder.TextField(self.spec.name, value=str(value.id)) - def get_html(self, value=None, context=None, other_values=None, **kwd): - context = context or {} - other_values = other_values or {} - return "
No display available for Metadata Files
" - def wrap(self, value, session): if value is None: return None From ec1438c9a0396cf0f370c66c14116b952bbd66a4 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:24:19 -0500 Subject: [PATCH 032/208] Avoid returning html in galaxy associated toolshed api --- lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 5ffbdcfe1b2..8871b893e1f 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -263,8 +263,8 @@ class ToolShedRepositoriesController(BaseAPIController): else: repository_data['repository'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', tsr_id])) repository_data['repository']['metadata'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', tsr_id, 'metadata'])) - repository_data['shed_conf'] = tool_util.build_shed_tool_conf_select_field(trans.app).get_html().replace('\n', '') - repository_data['panel_section_html'] = tool_panel_section_select_field.get_html(extra_attr={'style': 'width: 30em;'}).replace('\n', '') + repository_data['shed_conf'] = tool_util.build_shed_tool_conf_select_field(trans.app).to_dict() + repository_data['panel_section'] = tool_panel_section_select_field.to_dict() repository_data['panel_section_dict'] = tool_panel_section_dict for changeset, metadata in repository_data['repository']['metadata'].items(): if changeset not in tool_dependencies: From c20ddb096d36f2125dee2ec9aed9ba7fde2e04d4 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:26:30 -0500 Subject: [PATCH 033/208] Remove html helpers from second toolshed api controller --- lib/galaxy/webapps/galaxy/api/toolshed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/toolshed.py b/lib/galaxy/webapps/galaxy/api/toolshed.py index 48bc75f6ad1..a01950f91fd 100644 --- a/lib/galaxy/webapps/galaxy/api/toolshed.py +++ b/lib/galaxy/webapps/galaxy/api/toolshed.py @@ -251,8 +251,8 @@ class ToolShedController(BaseAPIController): else: repository_data['repository'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', repository_id])) repository_data['repository']['metadata'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', repository_id, 'metadata'])) - repository_data['shed_conf'] = tool_util.build_shed_tool_conf_select_field(trans.app).get_html().replace('\n', '') - repository_data['panel_section_html'] = tool_panel_section_select_field.get_html(extra_attr={'style': 'width: 30em;'}).replace('\n', '') + repository_data['shed_conf'] = tool_util.build_shed_tool_conf_select_field(trans.app).to_dict() + repository_data['panel_section'] = tool_panel_section_select_field.to_dict() repository_data['panel_section_dict'] = tool_panel_section_dict for changeset, metadata in repository_data['repository']['metadata'].items(): if changeset not in tool_dependencies: From 8d41a79ba12360357aae688600963e2f4a14f524 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:35:08 -0500 Subject: [PATCH 034/208] Remove html based input doc tests --- lib/galaxy/web/form_builder.py | 61 ---------------------------------- 1 file changed, 61 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index c0c49f9af0b..5940e24c1be 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -53,11 +53,6 @@ class BaseField(object): class TextField(BaseField): """ A standard text input box. - - >>> print(TextField( "foo" ).get_html()) - - >>> print(TextField( "bins", size=4, value="default" ).get_html()) - """ def __init__(self, name, size=None, value=None, **kwds): @@ -81,11 +76,6 @@ class TextField(BaseField): class PasswordField(BaseField): """ A password input box. text appears as "******" - - >>> print(PasswordField( "foo" ).get_html()) - - >>> print(PasswordField( "bins", size=4, value="default" ).get_html()) - """ def __init__(self, name, size=None, value=None, **kwds): @@ -110,11 +100,6 @@ class PasswordField(BaseField): class TextArea(BaseField): """ A standard text area box. - - >>> print(TextArea( "foo" ).get_html()) - - >>> print(TextArea( "bins", size="4x5", value="default" ).get_html()) - """ _DEFAULT_SIZE = "5x25" @@ -145,11 +130,6 @@ class TextArea(BaseField): class CheckboxField(BaseField): """ A checkbox (boolean input) - - >>> print(CheckboxField( "foo" ).get_html()) - - >>> print(CheckboxField( "bar", checked="yes" ).get_html()) - """ def __init__(self, name, checked=None, refresh_on_change=False, refresh_on_change_values=None, value=None, **kwds): @@ -195,11 +175,6 @@ class CheckboxField(BaseField): class FileField(BaseField): """ A file upload input. - - >>> print(FileField( "foo" ).get_html()) - - >>> print(FileField( "foo", ajax = True ).get_html()) - """ def __init__(self, name, value=None, ajax=False, **kwds): @@ -242,9 +217,6 @@ class GenomespaceFileField(BaseField): class HiddenField(BaseField): """ A hidden field. - - >>> print(HiddenField( "foo", 100 ).get_html()) - """ def __init__(self, name, value=None, **kwds): @@ -265,39 +237,6 @@ class HiddenField(BaseField): class SelectField(BaseField): """ A select field. - - >>> t = SelectField( "foo", multiple=True ) - >>> t.add_option( "tuti", 1 ) - >>> t.add_option( "fruity", "x" ) - >>> print(t.get_html()) - - - >>> t = SelectField( "bar" ) - >>> t.add_option( "automatic", 3 ) - >>> t.add_option( "bazooty", 4, selected=True ) - >>> print(t.get_html()) - - - >>> t = SelectField( "foo", display="radio" ) - >>> t.add_option( "tuti", 1 ) - >>> t.add_option( "fruity", "x" ) - >>> print(t.get_html()) -
-
- - >>> t = SelectField( "bar", multiple=True, display="checkboxes" ) - >>> t.add_option( "automatic", 3 ) - >>> t.add_option( "bazooty", 4, selected=True ) - >>> print(t.get_html()) -
-
-
""" def __init__(self, name, multiple=None, display=None, refresh_on_change=False, refresh_on_change_values=None, size=None, field_id=None, value=None, selectlist=None, **kwds): From 26e8d89a1a415d248aaa4c377b6587181bd77076 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:36:54 -0500 Subject: [PATCH 035/208] Move select builder to toolshed --- lib/galaxy/web/form_builder.py | 52 --------------------------- lib/tool_shed/util/repository_util.py | 52 ++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 5940e24c1be..5ddca726c1c 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -596,55 +596,3 @@ def get_suite(): import doctest import sys return doctest.DocTestSuite(sys.modules[__name__]) - - -# --------- Utility methods ----------------------------- -def build_select_field(trans, objs, label_attr, select_field_name, initial_value='none', - selected_value='none', refresh_on_change=False, multiple=False, display=None, size=None): - """ - Build a SelectField given a set of objects. The received params are: - - - objs: the set of objects used to populate the option list - - label_attr: the attribute of each obj (e.g., name, email, etc ) whose value is used to populate each option label. - - - If the string 'self' is passed as label_attr, each obj in objs is assumed to be a string, so the obj itself is used - - - select_field_name: the name of the SelectField - - initial_value: the value of the first option in the SelectField - allows for an option telling the user to select something - - selected_value: the value of the currently selected option - - refresh_on_change: True if the SelectField should perform a refresh_on_change - """ - if initial_value == 'none': - values = [initial_value] - else: - values = [] - for obj in objs: - if label_attr == 'self': - # Each obj is a string - values.append(obj) - else: - values.append(trans.security.encode_id(obj.id)) - if refresh_on_change: - refresh_on_change_values = values - else: - refresh_on_change_values = [] - select_field = SelectField(name=select_field_name, - multiple=multiple, - display=display, - refresh_on_change=refresh_on_change, - refresh_on_change_values=refresh_on_change_values, - size=size) - for obj in objs: - if label_attr == 'self': - # Each obj is a string - if str(selected_value) == str(obj): - select_field.add_option(obj, obj, selected=True) - else: - select_field.add_option(obj, obj) - else: - label = getattr(obj, label_attr) - if str(selected_value) == str(obj.id) or str(selected_value) == trans.security.encode_id(obj.id): - select_field.add_option(label, trans.security.encode_id(obj.id), selected=True) - else: - select_field.add_option(label, trans.security.encode_id(obj.id)) - return select_field diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 9e3c870d3fc..0b234d347d1 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -11,7 +11,6 @@ import tool_shed.dependencies.repository import tool_shed.util.metadata_util as metadata_util from galaxy import util from galaxy import web -from galaxy.web.form_builder import build_select_field from tool_shed.util import basic_util, common_util, encoding_util, hg_util from tool_shed.util.web_util import escape @@ -20,6 +19,57 @@ log = logging.getLogger(__name__) VALID_REPOSITORYNAME_RE = re.compile("^[a-z0-9\_]+$") +def build_select_field(trans, objs, label_attr, select_field_name, initial_value='none', + selected_value='none', refresh_on_change=False, multiple=False, display=None, size=None): + """ + Build a SelectField given a set of objects. The received params are: + + - objs: the set of objects used to populate the option list + - label_attr: the attribute of each obj (e.g., name, email, etc ) whose value is used to populate each option label. + + - If the string 'self' is passed as label_attr, each obj in objs is assumed to be a string, so the obj itself is used + + - select_field_name: the name of the SelectField + - initial_value: the value of the first option in the SelectField - allows for an option telling the user to select something + - selected_value: the value of the currently selected option + - refresh_on_change: True if the SelectField should perform a refresh_on_change + """ + if initial_value == 'none': + values = [initial_value] + else: + values = [] + for obj in objs: + if label_attr == 'self': + # Each obj is a string + values.append(obj) + else: + values.append(trans.security.encode_id(obj.id)) + if refresh_on_change: + refresh_on_change_values = values + else: + refresh_on_change_values = [] + select_field = SelectField(name=select_field_name, + multiple=multiple, + display=display, + refresh_on_change=refresh_on_change, + refresh_on_change_values=refresh_on_change_values, + size=size) + for obj in objs: + if label_attr == 'self': + # Each obj is a string + if str(selected_value) == str(obj): + select_field.add_option(obj, obj, selected=True) + else: + select_field.add_option(obj, obj) + else: + label = getattr(obj, label_attr) + if str(selected_value) == str(obj.id) or str(selected_value) == trans.security.encode_id(obj.id): + select_field.add_option(label, trans.security.encode_id(obj.id), selected=True) + else: + select_field.add_option(label, trans.security.encode_id(obj.id)) + return select_field + + def build_allow_push_select_field(trans, current_push_list, selected_value='none'): options = [] for user in trans.sa_session.query(trans.model.User): From 9b95e7e9519467187f2a9240c47551cb74e43291 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:39:03 -0500 Subject: [PATCH 036/208] Remove html helpers from form builder --- lib/galaxy/web/form_builder.py | 247 --------------------------------- 1 file changed, 247 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 5ddca726c1c..2b4cdc87ad6 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -29,10 +29,6 @@ class BaseField(object): self.optional = kwds.get('required', 'optional') == 'optional' self.help = kwds.get('helptext') - def get_html(self, prefix=""): - """Returns the html widget corresponding to the parameter""" - raise TypeError("Abstract Method") - def get_disabled_str(self, disabled=False): if disabled: return ' disabled="disabled"' @@ -59,11 +55,6 @@ class TextField(BaseField): super(TextField, self).__init__(name, value, **kwds) self.size = int(size or 10) - def get_html(self, prefix="", disabled=False): - value = unicodify(self.value or "") - return unicodify('' - % (prefix, self.name, self.size, escape(value, quote=True), self.get_disabled_str(disabled))) - def set_size(self, size): self.size = int(size) @@ -84,10 +75,6 @@ class PasswordField(BaseField): self.size = int(size or 10) self.value = value or "" - def get_html(self, prefix="", disabled=False): - return unicodify('' - % (prefix, self.name, self.size, escape(str(self.value), quote=True), self.get_disabled_str(disabled))) - def set_size(self, size): self.size = int(size) @@ -112,10 +99,6 @@ class TextArea(BaseField): self.cols = int(self.size[-1]) self.value = value or "" - def get_html(self, prefix="", disabled=False): - return unicodify('' - % (prefix, self.name, self.rows, self.cols, self.get_disabled_str(disabled), escape(str(self.value), quote=True))) - def set_size(self, rows, cols): self.rows = rows self.cols = cols @@ -145,15 +128,6 @@ class CheckboxField(BaseField): else: self.refresh_on_change_text = '' - def get_html(self, prefix="", disabled=False): - if self.checked: - checked_text = ' checked="checked"' - else: - checked_text = '' - id_name = prefix + self.name - return unicodify('' - % (id_name, id_name, checked_text, self.get_disabled_str(disabled), self.refresh_on_change_text, id_name, self.get_disabled_str(disabled))) - @staticmethod def is_checked(value): if value in [True, "true"]: @@ -183,15 +157,6 @@ class FileField(BaseField): self.ajax = ajax self.value = value - def get_html(self, prefix=""): - value_text = "" - if self.value: - value_text = ' value="%s"' % escape(str(self.value), quote=True) - ajax_text = "" - if self.ajax: - ajax_text = ' galaxy-ajax-upload="true"' - return unicodify('' % (prefix, self.name, ajax_text, value_text)) - class GenomespaceFileField(BaseField): """ @@ -201,14 +166,6 @@ class GenomespaceFileField(BaseField): self.name = name self.value = value or "" - def get_html(self, prefix=""): - return unicodify('' - ' ' - '' - 'Browse'.format(prefix, self.name, escape(str(self.value), quote=True))) - def to_dict(self): return dict(name=self.name, token_field=self.token_field) @@ -224,9 +181,6 @@ class HiddenField(BaseField): self.name = name self.value = value or "" - def get_html(self, prefix=""): - return unicodify('' % (prefix, self.name, escape(str(self.value), quote=True))) - def to_dict(self): d = super(HiddenField, self).to_dict() d['type'] = 'hidden' @@ -267,96 +221,6 @@ class SelectField(BaseField): def add_option(self, text, value, selected=False): self.options.append((text, value, selected)) - def get_html(self, prefix="", disabled=False, extra_attr=None): - if extra_attr is not None: - self.extra_attributes = ' %s' % ' '.join(['%s="%s"' % (k, escape(v)) for k, v in extra_attr.items()]) - else: - self.extra_attributes = '' - if self.display == "checkboxes": - return self.get_html_checkboxes(prefix, disabled) - elif self.display == "radio": - return self.get_html_radio(prefix, disabled) - else: - return self.get_html_default(prefix, disabled) - - def get_html_checkboxes(self, prefix="", disabled=False): - rval = [] - ctr = 0 - if len(self.options) > 1: - rval.append('
' % (prefix, self.name)) # placeholder for the insertion of the Select All/Unselect All buttons - for text, value, selected in self.options: - style = "" - text = unicodify(text) - escaped_value = escape(unicodify(value), quote=True) - uniq_id = "%s%s|%s" % (prefix, self.name, escaped_value) - if len(self.options) > 2 and ctr % 2 == 1: - style = " class=\"odd_row\"" - selected_text = "" - if selected: - selected_text = " checked='checked'" - rval.append('' - % (style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str(disabled), self.extra_attributes, uniq_id, escape(text, quote=True))) - ctr += 1 - return unicodify("\n".join(rval)) - - def get_html_radio(self, prefix="", disabled=False): - rval = [] - ctr = 0 - for text, value, selected in self.options: - style = "" - escaped_value = escape(str(value), quote=True) - uniq_id = "%s%s|%s" % (prefix, self.name, escaped_value) - if len(self.options) > 2 and ctr % 2 == 1: - style = " class=\"odd_row\"" - selected_text = "" - if selected: - selected_text = " checked='checked'" - rval.append('' - % (style, - prefix, - self.name, - self.refresh_on_change_text, - escaped_value, - uniq_id, - selected_text, - self.get_disabled_str(disabled), - self.extra_attributes, - uniq_id, - text)) - ctr += 1 - return unicodify("\n".join(rval)) - - def get_html_default(self, prefix="", disabled=False): - if self.multiple: - multiple = " multiple" - else: - multiple = "" - if self.size: - size = ' size="%s"' % str(self.size) - else: - size = '' - rval = [] - last_selected_value = "" - for text, value, selected in self.options: - if selected: - selected_text = " selected" - last_selected_value = value - if not isinstance(last_selected_value, string_types): - last_selected_value = str(last_selected_value) - else: - selected_text = "" - rval.append('' % (escape(unicodify(value), quote=True), selected_text, escape(unicodify(text), quote=True))) - if last_selected_value: - last_selected_value = ' last_selected_value="%s"' % escape(unicodify(last_selected_value), quote=True) - if self.field_id is not None: - id_string = ' id="%s"' % self.field_id - else: - id_string = '' - rval.insert(0, '') - return unicodify("\n".join(rval)) - def get_selected(self, return_label=False, return_value=False, multi=False): ''' Return the currently selected option's label, value or both as a tuple. For @@ -416,57 +280,6 @@ class AddressField(BaseField): self.select_address = None self.params = params - def get_html(self, disabled=False): - address_html = '' - add_ids = ['none'] - if self.user: - for a in self.user.addresses: - add_ids.append(str(a.id)) - add_ids.append('new') - self.select_address = SelectField(self.name, - refresh_on_change=True, - refresh_on_change_values=add_ids) - if self.value == 'none': - self.select_address.add_option('Select one', 'none', selected=True) - else: - self.select_address.add_option('Select one', 'none') - if self.user: - for a in self.user.addresses: - if not a.deleted: - if self.value == str(a.id): - self.select_address.add_option(a.desc, str(a.id), selected=True) - # Display this address - address_html += ''' -
- %s -
- ''' % a.get_html() - else: - self.select_address.add_option(a.desc, str(a.id)) - if self.value == 'new': - self.select_address.add_option('Add a new address', 'new', selected=True) - for field_name, label, help_text in self.fields(): - add_field = TextField(self.name + '_' + field_name, - 40, - restore_text(self.params.get(self.name + '_' + field_name, ''))) - address_html += ''' -
- - %s - ''' % (label, add_field.get_html(disabled=disabled)) - if help_text: - address_html += ''' -
- %s -
- ''' % help_text - address_html += ''' -
- ''' - else: - self.select_address.add_option('Add a new address', 'new') - return self.select_address.get_html(disabled=disabled) + address_html - def to_dict(self): d = super(AddressField, self).to_dict() d['type'] = 'select' @@ -488,21 +301,6 @@ class WorkflowField(BaseField): self.select_workflow = None self.params = params - def get_html(self, disabled=False): - self.select_workflow = SelectField(self.name) - if self.value == 'none': - self.select_workflow.add_option('Select one', 'none', selected=True) - else: - self.select_workflow.add_option('Select one', 'none') - if self.user: - for a in self.user.stored_workflows: - if not a.deleted: - if str(self.value) == str(a.id): - self.select_workflow.add_option(a.name, str(a.id), selected=True) - else: - self.select_workflow.add_option(a.name, str(a.id)) - return self.select_workflow.get_html(disabled=disabled) - def to_dict(self): d = super(WorkflowField, self).to_dict() d['type'] = 'select' @@ -524,31 +322,6 @@ class WorkflowMappingField(BaseField): self.params = params self.workflow_inputs = [] - def get_html(self, disabled=False): - self.select_workflow = SelectField(self.name, refresh_on_change=True) - workflow_inputs = [] - if self.value == 'none': - self.select_workflow.add_option('Select one', 'none', selected=True) - else: - self.select_workflow.add_option('Select one', 'none') - if self.user: - for a in self.user.stored_workflows: - if not a.deleted: - if str(self.value) == str(a.id): - self.select_workflow.add_option(a.name, str(a.id), selected=True) - else: - self.select_workflow.add_option(a.name, str(a.id)) - if self.value and self.value != 'none': - # Workflow selected. Find all inputs. - for workflow in self.user.stored_workflows: - if workflow.id == int(self.value): - for step in workflow.latest_workflow.steps: - if step.type == 'data_input': - if step.tool_inputs and "name" in step.tool_inputs: - workflow_inputs.append((step.tool_inputs['name'], TextField('%s_%s' % (self.name, step.id), 20))) - # Do something more appropriate here and allow selection of inputs - return self.select_workflow.get_html(disabled=disabled) + ''.join(['
%s
' % (s[0], s[1].get_html()) for s in workflow_inputs]) - class HistoryField(BaseField): def __init__(self, name, user=None, value=None, params=None, security=None, **kwds): @@ -560,26 +333,6 @@ class HistoryField(BaseField): self.select_history = None self.params = params - def get_html(self, disabled=False): - self.select_history = SelectField(self.name) - if self.value == 'none': - self.select_history.add_option('No Import', 'none', selected=True) - self.select_history.add_option('New History', 'new') - else: - self.select_history.add_option('No Import', 'none') - if self.value == 'new': - self.select_history.add_option('New History', 'new', selected=True) - else: - self.select_history.add_option('New History', 'new') - if self.user: - for a in self.user.histories: - if not a.deleted: - if str(self.value) == str(a.id): - self.select_history.add_option(a.name, str(a.id), selected=True) - else: - self.select_history.add_option(a.name, str(a.id)) - return self.select_history.get_html(disabled=disabled) - def to_dict(self): d = super(HistoryField, self).to_dict() d['type'] = 'select' From 4f80540e89834f3092e1d9e2ba2a993130e70953 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:41:26 -0500 Subject: [PATCH 037/208] Remove size handlers for html text fields --- lib/galaxy/web/form_builder.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 2b4cdc87ad6..17a67866e10 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -50,14 +50,6 @@ class TextField(BaseField): """ A standard text input box. """ - - def __init__(self, name, size=None, value=None, **kwds): - super(TextField, self).__init__(name, value, **kwds) - self.size = int(size or 10) - - def set_size(self, size): - self.size = int(size) - def to_dict(self): d = super(TextField, self).to_dict() d['type'] = 'text' @@ -68,16 +60,6 @@ class PasswordField(BaseField): """ A password input box. text appears as "******" """ - - def __init__(self, name, size=None, value=None, **kwds): - super(PasswordField, self).__init__(name, value, **kwds) - self.name = name - self.size = int(size or 10) - self.value = value or "" - - def set_size(self, size): - self.size = int(size) - def to_dict(self): d = super(PasswordField, self).to_dict() d['type'] = 'password' @@ -88,21 +70,6 @@ class TextArea(BaseField): """ A standard text area box. """ - _DEFAULT_SIZE = "5x25" - - def __init__(self, name, size=None, value=None, **kwds): - super(TextArea, self).__init__(name, value, **kwds) - self.name = name - size = size or self._DEFAULT_SIZE - self.size = size.split("x") - self.rows = int(self.size[0]) - self.cols = int(self.size[-1]) - self.value = value or "" - - def set_size(self, rows, cols): - self.rows = rows - self.cols = cols - def to_dict(self): d = super(TextArea, self).to_dict() d['type'] = 'text' From a840c4792295dce538e332496e2baeba11a2f2c0 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:42:02 -0500 Subject: [PATCH 038/208] Remove refresh on change handler for checkbox field --- lib/galaxy/web/form_builder.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 17a67866e10..e21a8cb16f9 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -86,14 +86,6 @@ class CheckboxField(BaseField): super(CheckboxField, self).__init__(name, value, **kwds) self.name = name self.checked = (checked is True) or (isinstance(checked, string_types) and (checked.lower() in ("yes", "true", "on"))) - self.refresh_on_change = refresh_on_change - self.refresh_on_change_values = refresh_on_change_values or [] - if self.refresh_on_change: - self.refresh_on_change_text = ' refresh_on_change="true" ' - if self.refresh_on_change_values: - self.refresh_on_change_text = '%s refresh_on_change_values="%s" ' % (self.refresh_on_change_text, ",".join(self.refresh_on_change_values)) - else: - self.refresh_on_change_text = '' @staticmethod def is_checked(value): From 96d3d27a59faa378dba4d4d9d50966b324751580 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:43:08 -0500 Subject: [PATCH 039/208] Remove html helper to disable input fields --- lib/galaxy/web/form_builder.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index e21a8cb16f9..0ff9b50e0d3 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -29,12 +29,6 @@ class BaseField(object): self.optional = kwds.get('required', 'optional') == 'optional' self.help = kwds.get('helptext') - def get_disabled_str(self, disabled=False): - if disabled: - return ' disabled="disabled"' - else: - return '' - def to_dict(self): return { 'name' : self.name, From 3bc44493367cd4f6e4c4efa5551463f35238f507 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:43:52 -0500 Subject: [PATCH 040/208] Remove unused file field from form builder --- lib/galaxy/web/form_builder.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 0ff9b50e0d3..5108bcfe617 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -99,18 +99,6 @@ class CheckboxField(BaseField): return d -class FileField(BaseField): - """ - A file upload input. - """ - - def __init__(self, name, value=None, ajax=False, **kwds): - super(FileField, self).__init__(name, value, **kwds) - self.name = name - self.ajax = ajax - self.value = value - - class GenomespaceFileField(BaseField): """ A genomspace file browser field. From ea9d53c5d8eef6bc67ceac55c2bc8a3b6a570a66 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:44:28 -0500 Subject: [PATCH 041/208] Remove unused genome space field from form builder --- lib/galaxy/web/form_builder.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 5108bcfe617..346002fd35b 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -99,19 +99,6 @@ class CheckboxField(BaseField): return d -class GenomespaceFileField(BaseField): - """ - A genomspace file browser field. - """ - def __init__(self, name, value=None): - self.name = name - self.value = value or "" - - def to_dict(self): - return dict(name=self.name, - token_field=self.token_field) - - class HiddenField(BaseField): """ A hidden field. From f77a47bf651366a259b7964658538e03f0900c73 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:46:10 -0500 Subject: [PATCH 042/208] Remove unused hidden field wrapper from form builder --- lib/galaxy/web/form_builder.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 346002fd35b..65641da2813 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -99,23 +99,6 @@ class CheckboxField(BaseField): return d -class HiddenField(BaseField): - """ - A hidden field. - """ - - def __init__(self, name, value=None, **kwds): - super(HiddenField, self).__init__(name, value, **kwds) - self.name = name - self.value = value or "" - - def to_dict(self): - d = super(HiddenField, self).to_dict() - d['type'] = 'hidden' - d['hidden'] = True - return d - - class SelectField(BaseField): """ A select field. From 16036fbe4f4166b53c31edde9e14256a5c87b554 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:48:35 -0500 Subject: [PATCH 043/208] Remove unused params parameter from addressfield wrapper --- lib/galaxy/web/form_builder.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 65641da2813..564477139fe 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -184,12 +184,10 @@ class AddressField(BaseField): ("country", "Country", ""), ("phone", "Phone", "")] - def __init__(self, name, user=None, value=None, params=None, security=None, **kwds): + def __init__(self, name, user=None, value=None, security=None, **kwds): super(AddressField, self).__init__(name, value, **kwds) self.user = user self.security = security - self.select_address = None - self.params = params def to_dict(self): d = super(AddressField, self).to_dict() From 09862dde84e0e9acbf7b09122cf22bcdc65759d5 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:49:30 -0500 Subject: [PATCH 044/208] Remove unused params parameter from workflow fields --- lib/galaxy/web/form_builder.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 564477139fe..66e96bb81fb 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -201,14 +201,11 @@ class AddressField(BaseField): class WorkflowField(BaseField): - def __init__(self, name, user=None, value=None, params=None, security=None, **kwds): + def __init__(self, name, user=None, value=None, security=None, **kwds): super(WorkflowField, self).__init__(name, value, **kwds) - self.name = name self.user = user self.value = value self.security = security - self.select_workflow = None - self.params = params def to_dict(self): d = super(WorkflowField, self).to_dict() From 82f56b004a440c7b57d4103e1b2b0359756b2c1e Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:51:11 -0500 Subject: [PATCH 045/208] Remove unused parameters from workflow mapping field --- lib/galaxy/web/form_builder.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 66e96bb81fb..8781b7423ea 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -219,14 +219,9 @@ class WorkflowField(BaseField): class WorkflowMappingField(BaseField): - def __init__(self, name, user=None, value=None, params=None, **kwd): - # DBTODO integrate this with the new __build_workflow approach in requests_common. As it is, not particularly useful. - self.name = name + def __init__(self, name, user=None, value=None, **kwd): + super(WorkflowMappingField, self).__init__(name, value, **kwds) self.user = user - self.value = value - self.select_workflow = None - self.params = params - self.workflow_inputs = [] class HistoryField(BaseField): From 0147816ecf6a00472a528f7fc672c9b45888287f Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 15:51:54 -0500 Subject: [PATCH 046/208] Remove unused parameters from history field --- lib/galaxy/web/form_builder.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 8781b7423ea..9fca5596c6a 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -225,14 +225,11 @@ class WorkflowMappingField(BaseField): class HistoryField(BaseField): - def __init__(self, name, user=None, value=None, params=None, security=None, **kwds): + def __init__(self, name, user=None, value=None, security=None, **kwds): super(HistoryField, self).__init__(name, value, **kwds) - self.name = name self.user = user self.value = value self.security = security - self.select_history = None - self.params = params def to_dict(self): d = super(HistoryField, self).to_dict() From d0565f1f5078e8e355a7f79c1f91411cd12070bb Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:04:59 -0500 Subject: [PATCH 047/208] Remove checkbox special case handling, only used in toolshed --- lib/galaxy/model/__init__.py | 8 +------- lib/galaxy/web/form_builder.py | 8 -------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4760b4ce1f9..b53a3340273 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4499,11 +4499,7 @@ class FormDefinition(object, Dictifiable): value = field.get('default', '') # Create the field widget field_widget = eval(field_type)(field_name) - if field_type in ['TextField', 'PasswordField']: - field_widget.set_size(40) - field_widget.value = value - elif field_type == 'TextArea': - field_widget.set_size(3, 40) + if field_type in ['TextField', 'TextArea', 'PasswordField', 'CheckboxField']: field_widget.value = value elif field_type in ['AddressField', 'WorkflowField', 'WorkflowMappingField', 'HistoryField']: field_widget.user = user @@ -4515,8 +4511,6 @@ class FormDefinition(object, Dictifiable): field_widget.add_option(option, option, selected=True) else: field_widget.add_option(option, option) - elif field_type == 'CheckboxField': - field_widget.set_checked(value) if field['required'] == 'required': req = 'Required' else: diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 9fca5596c6a..94c4e09f42e 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -79,19 +79,11 @@ class CheckboxField(BaseField): def __init__(self, name, checked=None, refresh_on_change=False, refresh_on_change_values=None, value=None, **kwds): super(CheckboxField, self).__init__(name, value, **kwds) self.name = name - self.checked = (checked is True) or (isinstance(checked, string_types) and (checked.lower() in ("yes", "true", "on"))) @staticmethod def is_checked(value): if value in [True, "true"]: return True - return isinstance(value, list) and ('__CHECKED__' in value or len(value) == 2) - - def set_checked(self, value): - if isinstance(value, string_types): - self.checked = value.lower() in ["yes", "true", "on"] - else: - self.checked = value def to_dict(self): d = super(CheckboxField, self).to_dict() From f8ef4a7bc14c9ad80a0fc7a27ffb89fadefc14eb Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:09:38 -0500 Subject: [PATCH 048/208] Remove template form helper for libraries --- lib/galaxy/model/__init__.py | 143 ----------------------------------- 1 file changed, 143 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b53a3340273..0cda32492a9 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2675,24 +2675,6 @@ class Library(object, Dictifiable, HasName): return None, inherited return None, inherited - def get_template_widgets(self, trans, get_contents=True): - # See if we have any associated templates - the returned value for - # inherited is not applicable at the library level. The get_contents - # param is passed by callers that are inheriting a template - these - # are usually new library datsets for which we want to include template - # fields on the upload form, but not necessarily the contents of the - # inherited template saved for the parent. - info_association, inherited = self.get_info_association() - if info_association: - template = info_association.template - if get_contents: - # See if we have any field contents - info = info_association.info - if info: - return template.get_widgets(trans.user, contents=info.content) - return template.get_widgets(trans.user) - return [] - def get_access_roles(self, trans): roles = [] for lp in self.actions: @@ -2742,32 +2724,6 @@ class LibraryFolder(object, Dictifiable, HasName): return self.library_root[0].get_info_association(inherited=True) return None, inherited - def get_template_widgets(self, trans, get_contents=True): - # See if we have any associated templates. The get_contents - # param is passed by callers that are inheriting a template - these - # are usually new library datsets for which we want to include template - # fields on the upload form. - info_association, inherited = self.get_info_association() - if info_association: - if inherited: - template = info_association.template.current.latest_form - else: - template = info_association.template - # See if we have any field contents, but only if the info_association was - # not inherited ( we do not want to display the inherited contents ). - # (gvk: 8/30/10) Based on conversations with Dan, we agreed to ALWAYS inherit - # contents. We'll use this behavior until we hear from the community that - # contents should not be inherited. If we don't hear anything for a while, - # eliminate the old commented out behavior. - # if not inherited and get_contents: - if get_contents: - info = info_association.info - if info: - return template.get_widgets(trans.user, info.content) - else: - return template.get_widgets(trans.user) - return [] - @property def activatable_library_datasets(self): # This needs to be a list @@ -3044,33 +3000,6 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): rval['metadata_' + name] = val return rval - def get_template_widgets(self, trans, get_contents=True): - # See if we have any associated templatesThe get_contents - # param is passed by callers that are inheriting a template - these - # are usually new library datsets for which we want to include template - # fields on the upload form, but not necessarily the contents of the - # inherited template saved for the parent. - info_association, inherited = self.get_info_association() - if info_association: - if inherited: - template = info_association.template.current.latest_form - else: - template = info_association.template - # See if we have any field contents, but only if the info_association was - # not inherited ( we do not want to display the inherited contents ). - # (gvk: 8/30/10) Based on conversations with Dan, we agreed to ALWAYS inherit - # contents. We'll use this behavior until we hear from the community that - # contents should not be inherited. If we don't hear anything for a while, - # eliminate the old commented out behavior. - # if not inherited and get_contents: - if get_contents: - info = info_association.info - if info: - return template.get_widgets(trans.user, info.content) - else: - return template.get_widgets(trans.user) - return [] - def templates_dict(self, use_name=False): """ Returns a dict of template info @@ -4450,78 +4379,6 @@ class FormDefinition(object, Dictifiable): gridfields[i] = f return gridfields - def get_widgets(self, user, contents={}, **kwd): - ''' - Return the list of widgets that comprise a form definition, - including field contents if any. - ''' - params = Params(kwd) - widgets = [] - for index, field in enumerate(self.fields): - field_type = field['type'] - if 'name' in field: - field_name = field['name'] - else: - # Default to names like field_0, field_1, etc for backward compatibility - # (not sure this is necessary)... - field_name = 'field_%i' % index - # Determine the value of the field - if field_name in kwd: - # The form was submitted via refresh_on_change - if field_type == 'CheckboxField': - value = CheckboxField.is_checked(params.get(field_name, False)) - else: - value = restore_text(params.get(field_name, '')) - elif contents: - try: - # This field has a saved value. - value = str(contents[field['name']]) - except Exception: - # If there was an error getting the saved value, we'll still - # display the widget, but it will be empty. - if field_type == 'AddressField': - value = 'none' - elif field_type == 'CheckboxField': - # Since we do not have contents, set checkbox value to False - value = False - else: - # Set other field types to empty string - value = '' - else: - # If none of the above, then leave the field empty - if field_type == 'AddressField': - value = 'none' - elif field_type == 'CheckboxField': - # Since we do not have contents, set checkbox value to False - value = False - else: - # Set other field types to the default value of the field - value = field.get('default', '') - # Create the field widget - field_widget = eval(field_type)(field_name) - if field_type in ['TextField', 'TextArea', 'PasswordField', 'CheckboxField']: - field_widget.value = value - elif field_type in ['AddressField', 'WorkflowField', 'WorkflowMappingField', 'HistoryField']: - field_widget.user = user - field_widget.value = value - field_widget.params = params - elif field_type == 'SelectField': - for option in field['selectlist']: - if option == value: - field_widget.add_option(option, option, selected=True) - else: - field_widget.add_option(option, option) - if field['required'] == 'required': - req = 'Required' - else: - req = 'Optional' - if field['helptext']: - helptext = '%s (%s)' % (field['helptext'], req) - else: - helptext = '(%s)' % req - widgets.append(dict(label=field['label'], widget=field_widget, helptext=helptext)) - return widgets - class FormDefinitionCurrent(object): def __init__(self, form_definition=None): From 797fd8286f2db4656a6d9ba8a720a4f1f12e0b3a Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:14:14 -0500 Subject: [PATCH 049/208] Remove get widget helper --- .../webapps/galaxy/api/library_contents.py | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index 70f51b5a88d..551575b4a8e 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -300,18 +300,12 @@ class LibraryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary roles = kwd.get('roles', '') is_admin = trans.user_is_admin() current_user_roles = trans.get_current_user_roles() - widgets = [] - info_association, inherited = None, None - template_id = "None" if replace_id not in ['', None, 'None']: replace_dataset = trans.sa_session.query(trans.app.model.LibraryDataset).get(trans.security.decode_id(replace_id)) self._check_access(trans, is_admin, replace_dataset, current_user_roles) self._check_modify(trans, is_admin, replace_dataset, current_user_roles) library = replace_dataset.folder.parent_library folder = replace_dataset.folder - info_association, inherited = replace_dataset.library_dataset_dataset_association.get_info_association() - if info_association and (not(inherited) or info_association.inheritable): - widgets = replace_dataset.library_dataset_dataset_association.get_template_widgets(trans) # The name is stored - by the time the new ldda is created, replace_dataset.name # will point to the new ldda, not the one it's replacing. if not last_used_build: @@ -342,56 +336,9 @@ class LibraryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary if error: return 400, message else: - # See if we have any inherited templates. - if not info_association: - info_association, inherited = folder.get_info_association(inherited=True) - if info_association and info_association.inheritable: - template_id = str(info_association.template.id) - widgets = folder.get_template_widgets(trans, get_contents=True) - processed_widgets = [] - # The list of widgets may include an AddressField which we need to save if it is new - for index, widget_dict in enumerate(widgets): - widget = widget_dict['widget'] - if isinstance(widget, AddressField): - value = kwd.get(widget.name, '') - if value == 'new': - if self.field_param_values_ok(widget.name, 'AddressField', **kwd): - # Save the new address - address = trans.app.model.UserAddress(user=trans.user) - self.save_widget_field(trans, address, widget.name, **kwd) - widget.value = str(address.id) - widget_dict['widget'] = widget - processed_widgets.append(widget_dict) - # It is now critical to update the value of 'field_%i', replacing the string - # 'new' with the new address id. This is necessary because the upload_dataset() - # method below calls the handle_library_params() method, which does not parse the - # widget fields, it instead pulls form values from kwd. See the FIXME comments in the - # handle_library_params() method, and the CheckboxField code in the next conditional. - kwd[widget.name] = str(address.id) - else: - # The invalid address won't be saved, but we cannot display error - # messages on the upload form due to the ajax upload already occurring. - # When we re-engineer the upload process ( currently under way ), we - # will be able to check the form values before the ajax upload occurs - # in the background. For now, we'll do nothing... - pass - elif isinstance(widget, CheckboxField): - # We need to check the value from kwd since util.Params would have munged the list if - # the checkbox is checked. - value = kwd.get(widget.name, '') - if CheckboxField.is_checked(value): - widget.value = 'true' - widget_dict['widget'] = widget - processed_widgets.append(widget_dict) - kwd[widget.name] = 'true' - else: - processed_widgets.append(widget_dict) - widgets = processed_widgets created_outputs_dict = self._upload_dataset(trans, library_id=trans.security.encode_id(library.id), folder_id=trans.security.encode_id(folder.id), - template_id=template_id, - widgets=widgets, replace_dataset=replace_dataset, **kwd) if created_outputs_dict: From df5a9530e0dbdf94d05bf5ee25ca722278e927d6 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:17:57 -0500 Subject: [PATCH 050/208] Remove association for template helpers --- lib/galaxy/model/__init__.py | 81 +----------------------------------- 1 file changed, 1 insertion(+), 80 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0cda32492a9..a0ae115752d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2667,14 +2667,6 @@ class Library(object, Dictifiable, HasName): active_folders.extend(self.get_active_folders(active_folder, folders)) return sort_by_attr(active_folders, 'id') - def get_info_association(self, restrict=False, inherited=False): - if self.info_association: - if not inherited or self.info_association[0].inheritable: - return self.info_association[0], inherited - else: - return None, inherited - return None, inherited - def get_access_roles(self, trans): roles = [] for lp in self.actions: @@ -2705,25 +2697,6 @@ class LibraryFolder(object, Dictifiable, HasName): folder.order_id = self.item_count self.item_count += 1 - def get_info_association(self, restrict=False, inherited=False): - # If restrict is True, we will return this folder's info_association, not inheriting. - # If restrict is False, we'll return the next available info_association in the - # inheritable hierarchy if it is "inheritable". True is also returned if the - # info_association was inherited and False if not. This enables us to eliminate - # displaying any contents of the inherited template. - if self.info_association: - if not inherited or self.info_association[0].inheritable: - return self.info_association[0], inherited - else: - return None, inherited - if restrict: - return None, inherited - if self.parent: - return self.parent.get_info_association(inherited=True) - if self.library_root: - return self.library_root[0].get_info_association(inherited=True) - return None, inherited - @property def activatable_library_datasets(self): # This needs to be a list @@ -2731,13 +2704,6 @@ class LibraryFolder(object, Dictifiable, HasName): def to_dict(self, view='collection', value_mapper=None): rval = super(LibraryFolder, self).to_dict(view=view, value_mapper=value_mapper) - info_association, inherited = self.get_info_association() - if info_association: - if inherited: - template = info_association.template.current.latest_form - else: - template = info_association.template - rval['data_template'] = template.name rval['library_path'] = self.library_path rval['parent_library_id'] = self.parent_library.id return rval @@ -2811,15 +2777,6 @@ class LibraryDataset(object): # display in other objects, we can't use the simpler method used by # other model classes. ldda = self.library_dataset_dataset_association - template_data = {} - for temp_info in ldda.info_association: - template = temp_info.template - content = temp_info.info.content - tmp_dict = {} - for field in template.fields: - tmp_dict[field['label']] = content[field['name']] - template_data[template.name] = tmp_dict - rval = dict(id=self.id, ldda_id=ldda.id, parent_library_id=self.folder.parent_library.id, @@ -2837,8 +2794,7 @@ class LibraryDataset(object): genome_build=ldda.dbkey, misc_info=ldda.info, misc_blurb=ldda.blurb, - peek=(lambda ldda: ldda.display_peek() if ldda.peek and ldda.peek != 'no peek' else None)(ldda), - template_data=template_data) + peek=(lambda ldda: ldda.display_peek() if ldda.peek and ldda.peek != 'no peek' else None)(ldda)) if ldda.dataset.uuid is None: rval['uuid'] = None else: @@ -2943,19 +2899,6 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): def has_manage_permissions_roles(self, trans): return self.dataset.has_manage_permissions_roles(trans) - def get_info_association(self, restrict=False, inherited=False): - # If restrict is True, we will return this ldda's info_association whether it - # exists or not ( in which case None will be returned ). If restrict is False, - # we'll return the next available info_association in the inheritable hierarchy. - # True is also returned if the info_association was inherited, and False if not. - # This enables us to eliminate displaying any contents of the inherited template. - # SM: Accessing self.info_association can cause a query to be emitted - if self.info_association: - return self.info_association[0], inherited - if restrict: - return None, inherited - return self.library_dataset.folder.get_info_association(inherited=True) - def to_dict(self, view='collection'): # Since this class is a proxy to rather complex attributes we want to # display in other objects, we can't use the simpler method used by @@ -3000,28 +2943,6 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): rval['metadata_' + name] = val return rval - def templates_dict(self, use_name=False): - """ - Returns a dict of template info - """ - # TODO: Should have a method that allows names and labels to be returned together in a structured way - template_data = {} - for temp_info in self.info_association: - template = temp_info.template - content = temp_info.info.content - tmp_dict = {} - for field in template.fields: - if use_name: - name = field['name'] - else: - name = field['label'] - tmp_dict[name] = content.get(field['name']) - template_data[template.name] = tmp_dict - return template_data - - def templates_json(self, use_name=False): - return json.dumps(self.templates_dict(use_name=use_name)) - class ExtendedMetadata(object): def __init__(self, data): From 4d1df8c4566e539e4c192867eea9cfc761191a6b Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:20:29 -0500 Subject: [PATCH 051/208] Remove unused checkbox parameters --- lib/galaxy/web/form_builder.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 94c4e09f42e..a731093e4e4 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -75,11 +75,6 @@ class CheckboxField(BaseField): """ A checkbox (boolean input) """ - - def __init__(self, name, checked=None, refresh_on_change=False, refresh_on_change_values=None, value=None, **kwds): - super(CheckboxField, self).__init__(name, value, **kwds) - self.name = name - @staticmethod def is_checked(value): if value in [True, "true"]: From 296649937ff045734480a4b673902f5921cf3ad5 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:22:51 -0500 Subject: [PATCH 052/208] Remove unused on change handler for galaxy select fields --- lib/galaxy/web/form_builder.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index a731093e4e4..b7ced2bc5b2 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -93,12 +93,9 @@ class SelectField(BaseField): def __init__(self, name, multiple=None, display=None, refresh_on_change=False, refresh_on_change_values=None, size=None, field_id=None, value=None, selectlist=None, **kwds): super(SelectField, self).__init__(name, value, **kwds) - self.name = name self.field_id = field_id self.multiple = multiple or False self.selectlist = selectlist or [] - self.value = value - self.size = size self.options = list() if display == "checkboxes": assert multiple, "Checkbox display only supported for multiple select" @@ -107,14 +104,6 @@ class SelectField(BaseField): elif display is not None: raise Exception("Unknown display type: %s" % display) self.display = display - self.refresh_on_change = refresh_on_change - self.refresh_on_change_values = refresh_on_change_values or [] - if self.refresh_on_change: - self.refresh_on_change_text = ' refresh_on_change="true"' - if self.refresh_on_change_values: - self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % (self.refresh_on_change_text, escape(",".join(self.refresh_on_change_values), quote=True)) - else: - self.refresh_on_change_text = '' def add_option(self, text, value, selected=False): self.options.append((text, value, selected)) From 31607438e7492467e1d689060007dfaed45e5fcd Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:23:11 -0500 Subject: [PATCH 053/208] Remove unused select value helper --- lib/galaxy/web/form_builder.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index b7ced2bc5b2..f7656e11fae 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -108,34 +108,6 @@ class SelectField(BaseField): def add_option(self, text, value, selected=False): self.options.append((text, value, selected)) - def get_selected(self, return_label=False, return_value=False, multi=False): - ''' - Return the currently selected option's label, value or both as a tuple. For - multi-select lists, a list is returned. - ''' - if multi: - selected_options = [] - for label, value, selected in self.options: - if selected: - if return_label and return_value: - if multi: - selected_options.append((label, value)) - else: - return (label, value) - elif return_label: - if multi: - selected_options.append(label) - else: - return label - elif return_value: - if multi: - selected_options.append(value) - else: - return value - if multi: - return selected_options - return None - def to_dict(self): d = super(SelectField, self).to_dict() d['type'] = 'select' From 3eed3a4df058c1674ec3d4682c8365d45f4b2386 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:24:53 -0500 Subject: [PATCH 054/208] Remove unused mako templates for custom form building --- templates/common/select_template.mako | 76 ---------- templates/common/template_common.mako | 200 -------------------------- 2 files changed, 276 deletions(-) delete mode 100644 templates/common/select_template.mako delete mode 100644 templates/common/template_common.mako diff --git a/templates/common/select_template.mako b/templates/common/select_template.mako deleted file mode 100644 index 7044f5a693f..00000000000 --- a/templates/common/select_template.mako +++ /dev/null @@ -1,76 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> - -<% - in_library = form_type == trans.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE - in_sample_tracking = form_type == trans.model.FormDefinition.types.RUN_DETAILS_TEMPLATE - if in_library: - # If rendering for a library folder or dataset, inheritance is set by the user, while - # rendering for a RequestType, the template is always available to samples. - from galaxy.web.form_builder import CheckboxField - inheritable_check_box = CheckboxField( 'inheritable' ) -%> - -

- - -%if message: - ${render_msg( message, status )} -%endif - -
-
Select a template for the ${item_desc} '${util.unicodify( item_name )}'
-
- %if form_type == trans.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE: -
- %elif form_type == trans.model.FormDefinition.types.RUN_DETAILS_TEMPLATE: - - %endif -
- - ${form_id_select_field.get_html()} -
- % if form_type == trans.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE and item_type in [ 'library', 'folder' ]: -
- %if inheritable_checked: - <% inheritable_check_box.checked = True %> - %endif - ${inheritable_check_box.get_html()} - -
- Check if you want this template to be used by other folders and datasets contained within this ${item_desc} -
-
- %endif -
- -
-
-
-
-

-%if form_id_select_field.get_selected( return_label=True, return_value=True ) != ('Select one', 'none'): -

-
Layout of selected template
-
-
- %for i, field in enumerate( widgets ): -
- - ${field[ 'widget' ].get_html( disabled=True )} -
- ${field[ 'helptext' ]} -
-
-
- %endfor -
-
-
-%endif diff --git a/templates/common/template_common.mako b/templates/common/template_common.mako deleted file mode 100644 index 1bd4c00e838..00000000000 --- a/templates/common/template_common.mako +++ /dev/null @@ -1,200 +0,0 @@ -<%def name="render_template_field( field, render_as_hidden=False )"> - <% - from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField - - widget = field[ 'widget' ] - has_contents = False - label = field[ 'label' ] - value = '' - if isinstance( widget, TextArea ) and widget.value: - has_contents = True - if render_as_hidden: - value = widget.value - else: - value = '
%s
' % widget.value - elif isinstance( widget, TextField ) and widget.value: - has_contents = True - value = widget.value - elif isinstance( widget, SelectField ) and widget.options: - for option_label, option_value, selected in widget.options: - if selected: - has_contents = True - value = option_value - elif isinstance( widget, CheckboxField ) and widget.checked: - has_contents = True - if render_as_hidden: - value = 'true' - else: - value = 'checked' - elif isinstance( widget, WorkflowField ) and str( widget.value ).lower() not in [ 'none' ]: - has_contents = True - if render_as_hidden: - value = widget.value - else: - workflow_user = widget.user - if workflow_user: - for workflow in workflow_user.stored_workflows: - if not workflow.deleted and str( widget.value ) == str( workflow.id ): - value = workflow.name - break - else: - # If we didn't find the selected workflow option above, we'll just print the value - value = widget.value - elif isinstance( widget, WorkflowMappingField ) and str( widget.value ).lower() not in [ 'none' ]: - has_contents = True - if render_as_hidden: - value = widget.value - else: - workflow_user = widget.user - if workflow_user: - for workflow in workflow_user.stored_workflows: - if not workflow.deleted and str( widget.value ) == str( workflow.id ): - value = workflow.name - break - else: - # If we didn't find the selected workflow option above, we'll just print the value - value = widget.value - elif isinstance( widget, HistoryField ) and str( widget.value ).lower() not in [ 'none' ]: - has_contents = True - if render_as_hidden: - value = widget.value - else: - history_user = widget.user - if history_user: - for history in history_user.histories: - if not history.deleted and str( widget.value ) == str( history.id ): - value = util.unicodify( history.name ) - break - else: - # If we didn't find the selected workflow option above, we'll just print the value - value = widget.value - elif isinstance( widget, AddressField ) and str( widget.value ).lower() not in [ 'none' ]: - has_contents = True - if render_as_hidden: - value = widget.value - else: - address = trans.sa_session.query( trans.model.UserAddress ).get( int( widget.value ) ) - label = address.desc - value = address.get_html() - %> - %if has_contents: - % if render_as_hidden: - - %else: -
- - ${value} -
- ${field[ 'helptext' ]} -
-
-
- %endif - %endif - - -<%def name="render_template_fields( cntrller, item_type, widgets, widget_fields_have_contents, request_type_id=None, sample_id=None, library_id=None, folder_id=None, ldda_id=None, info_association=None, inherited=False, editable=True )"> - <% - in_library = False - in_sample_tracking = False - - if item_type == 'library': - item = trans.sa_session.query( trans.app.model.Library ).get( trans.security.decode_id( library_id ) ) - elif item_type == 'folder': - item = trans.sa_session.query( trans.app.model.LibraryFolder ).get( trans.security.decode_id( folder_id ) ) - elif item_type == 'ldda': - item = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id( ldda_id ) ) - elif item_type == 'request_type': - item = trans.sa_session.query( trans.app.model.RequestType ).get( trans.security.decode_id( request_type_id ) ) - elif item_type == 'sample': - item = trans.sa_session.query( trans.app.model.Sample ).get( trans.security.decode_id( sample_id ) ) - - if cntrller in [ 'library', 'library_admin' ]: - in_library = True - template_section_title = 'Other information' - form_type = trans.model.FormDefinition.types.LIBRARY_INFO_TEMPLATE - if trans.user_is_admin() and cntrller == 'library_admin': - can_modify = True - elif cntrller == 'library': - can_modify = trans.app.security_agent.can_modify_library_item( trans.get_current_user_roles(), item ) - else: - can_modify = False - elif cntrller in [ 'requests_admin', 'requests', 'request_type' ]: - in_sample_tracking = True - template_section_title = 'Run details' - form_type = trans.model.FormDefinition.types.RUN_DETAILS_TEMPLATE - %> - %if ( in_sample_tracking and editable ) or ( in_library and editable and can_modify ): -

-

-
- -
- %if in_library and info_association and inherited and can_modify: - ## "inherited" will be true only if the info_association is not associated with the current item, - ## which means that the currently display template has not yet been saved for the current item. - Select a different template - %elif in_library and info_association and not inherited and can_modify: - Edit template - Unuse template - %if item_type not in [ 'ldda', 'library_dataset' ]: - %if info_association.inheritable: - Dis-inherit template - %else: - Inherit template - %endif - %endif - %elif in_sample_tracking: - Select a different template - Edit template - Unuse template - %endif -
-
-
- %if in_library and inherited: -
- - - This is an inherited template and is not required to be used with this ${item_type}. You can - Select a different template - or fill in the desired fields and save this one. This template will not be associated with this ${item_type} until you click the Save button. - - -
- %endif - %if in_library: -
- %elif in_sample_tracking: - - %endif - %for i, field in enumerate( widgets ): -
- - ${field[ 'widget' ].get_html()} -
- ${field[ 'helptext' ]} -
-
-
- %endfor -
- -
-
-
-
-

- %elif widget_fields_have_contents: -

-

-
Other information about ${ util.unicodify( item.name )}
-
- %for i, field in enumerate( widgets ): - ${render_template_field( field )} - %endfor -
-
-

- %endif - From 26677708e14b5e4ae23e4dfea5b105b74bc4b134 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:26:19 -0500 Subject: [PATCH 055/208] Remove unused form template loader --- lib/galaxy/web/framework/webapp.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index 37eb57365c7..91491195bf4 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -872,15 +872,6 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, """ return self.show_message(message, 'warning', refresh_frames, use_panels=use_panels, active_view=active_view) - def show_form(self, form, header=None, template="form.mako", use_panels=False, active_view=""): - """ - Convenience method for displaying a simple page with a single HTML - form. - """ - return self.fill_template(template, form=form, header=header, - use_panels=(form.use_panels or use_panels), - active_view=active_view) - @property def session_csrf_token(self): token = '' From 07ce743b8df873a348d883e202c5f3f45315aceb Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:27:31 -0500 Subject: [PATCH 056/208] Remove form.mako --- templates/form.mako | 112 -------------------------------------------- 1 file changed, 112 deletions(-) delete mode 100644 templates/form.mako diff --git a/templates/form.mako b/templates/form.mako deleted file mode 100644 index be0eab37d8a..00000000000 --- a/templates/form.mako +++ /dev/null @@ -1,112 +0,0 @@ -<%! - def inherit(context): - if context.get('use_panels'): - if context.get('webapp'): - app_name = context.get('webapp') - elif context.get('app'): - app_name = context.get('app').name - else: - app_name = 'galaxy' - return '/webapps/%s/base_panels.mako' % app_name - else: - return '/base.mako' -%> -<%inherit file="${inherit(context)}"/> -<% _=n_ %> - -<%def name="init()"> -<% - self.has_left_panel=False - self.has_right_panel=False - self.active_view=active_view - self.message_box_visible=False -%> - - - -<%def name="title()">${form.title | h} - -<%def name="javascripts()"> - ${parent.javascripts()} - ${h.js("libs/jquery/jquery.autocomplete")} - - - -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css("autocomplete_tagging")} - - -<%def name="center_panel()"> - ${render_form( )} - - -<%def name="body()"> - ${render_form( )} - - -<%def name="render_form()"> - %if header: - ${header} - %endif - -

-
${util.unicodify( form.title ) | h }
-
- <% - has_file_input = False - for input in form.inputs: - if input.type == 'file': - has_file_input = True - break - %> -
- %for input in form.inputs: - <% - cls = "form-row" - if input.error: - cls += " form-row-error" - %> -
- %if input.use_label: - - %endif -
- %if input.type == 'textarea': - - %elif input.type == 'select': - - %else: - - %endif -
- %if input.error: -
${input.error | h }
- %endif - %if input.help: -
- ${input.help | h} -
- %endif -
-
- %endfor -
-
-
-
- From d489aea241a27dfc8a15875a0cb37c2f54f2eacc Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:34:52 -0500 Subject: [PATCH 057/208] Remove unused refresh on change parsing for tool version selection field --- lib/galaxy/tools/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 4d872d4fc60..a7bb95142e4 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -309,11 +309,9 @@ class ToolBox(BaseGalaxyToolBox): def __build_tool_version_select_field(self, tools, tool_id, set_selected): """Build a SelectField whose options are the ids for the received list of tools.""" options = [] - refresh_on_change_values = [] for tool in tools: options.insert(0, (tool.version, tool.id)) - refresh_on_change_values.append(tool.id) - select_field = SelectField(name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values) + select_field = SelectField(name='tool_id') for option_tup in options: selected = set_selected and option_tup[1] == tool_id if selected: From 91c05859b6f9fc9fefc3b4fca3f1dc12cac9a176 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:50:07 -0500 Subject: [PATCH 058/208] Move remaining form builder to toolshed --- lib/tool_shed/util/repository_util.py | 52 +-------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 0b234d347d1..9e3c870d3fc 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -11,6 +11,7 @@ import tool_shed.dependencies.repository import tool_shed.util.metadata_util as metadata_util from galaxy import util from galaxy import web +from galaxy.web.form_builder import build_select_field from tool_shed.util import basic_util, common_util, encoding_util, hg_util from tool_shed.util.web_util import escape @@ -19,57 +20,6 @@ log = logging.getLogger(__name__) VALID_REPOSITORYNAME_RE = re.compile("^[a-z0-9\_]+$") -def build_select_field(trans, objs, label_attr, select_field_name, initial_value='none', - selected_value='none', refresh_on_change=False, multiple=False, display=None, size=None): - """ - Build a SelectField given a set of objects. The received params are: - - - objs: the set of objects used to populate the option list - - label_attr: the attribute of each obj (e.g., name, email, etc ) whose value is used to populate each option label. - - - If the string 'self' is passed as label_attr, each obj in objs is assumed to be a string, so the obj itself is used - - - select_field_name: the name of the SelectField - - initial_value: the value of the first option in the SelectField - allows for an option telling the user to select something - - selected_value: the value of the currently selected option - - refresh_on_change: True if the SelectField should perform a refresh_on_change - """ - if initial_value == 'none': - values = [initial_value] - else: - values = [] - for obj in objs: - if label_attr == 'self': - # Each obj is a string - values.append(obj) - else: - values.append(trans.security.encode_id(obj.id)) - if refresh_on_change: - refresh_on_change_values = values - else: - refresh_on_change_values = [] - select_field = SelectField(name=select_field_name, - multiple=multiple, - display=display, - refresh_on_change=refresh_on_change, - refresh_on_change_values=refresh_on_change_values, - size=size) - for obj in objs: - if label_attr == 'self': - # Each obj is a string - if str(selected_value) == str(obj): - select_field.add_option(obj, obj, selected=True) - else: - select_field.add_option(obj, obj) - else: - label = getattr(obj, label_attr) - if str(selected_value) == str(obj.id) or str(selected_value) == trans.security.encode_id(obj.id): - select_field.add_option(label, trans.security.encode_id(obj.id), selected=True) - else: - select_field.add_option(label, trans.security.encode_id(obj.id)) - return select_field - - def build_allow_push_select_field(trans, current_push_list, selected_value='none'): options = [] for user in trans.sa_session.query(trans.model.User): From 8ccdee9690eac96da1b12e5b6b22c72509b90af3 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:52:20 -0500 Subject: [PATCH 059/208] Link toolshed form builder to repository utilites --- lib/tool_shed/util/repository_util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 9e3c870d3fc..69e3c27d8b0 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -11,8 +11,7 @@ import tool_shed.dependencies.repository import tool_shed.util.metadata_util as metadata_util from galaxy import util from galaxy import web -from galaxy.web.form_builder import build_select_field -from tool_shed.util import basic_util, common_util, encoding_util, hg_util +from tool_shed.util import basic_util, common_util, encoding_util, form_util, hg_util from tool_shed.util.web_util import escape log = logging.getLogger(__name__) @@ -25,7 +24,7 @@ def build_allow_push_select_field(trans, current_push_list, selected_value='none for user in trans.sa_session.query(trans.model.User): if user.username not in current_push_list: options.append(user) - return build_select_field(trans, + return form_util.build_select_field(trans, objs=options, label_attr='username', select_field_name='allow_push', From ec16aadccc9a476d5f2936953fe2975246b460ab Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:53:53 -0500 Subject: [PATCH 060/208] Link toolshed form builder with grid utilities --- lib/tool_shed/grids/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/grids/util.py b/lib/tool_shed/grids/util.py index 810a0ab42b9..72536e2c7a1 100644 --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,6 +1,6 @@ import logging -from galaxy.web.form_builder import SelectField +from tool_shed.util.form_util import SelectField from tool_shed.util import hg_util, metadata_util log = logging.getLogger(__name__) From a726eba67bd0a762346cec3cd2338481085b142d Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 16:54:18 -0500 Subject: [PATCH 061/208] Add form builder for toolshed --- lib/tool_shed/util/form_util.py | 265 ++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 lib/tool_shed/util/form_util.py diff --git a/lib/tool_shed/util/form_util.py b/lib/tool_shed/util/form_util.py new file mode 100644 index 00000000000..631897cb91d --- /dev/null +++ b/lib/tool_shed/util/form_util.py @@ -0,0 +1,265 @@ +""" +Classes for generating HTML forms +""" +from __future__ import print_function + +import logging +from cgi import escape + +from six import string_types + +from galaxy.util import ( + asbool, + restore_text, + unicodify +) + +log = logging.getLogger(__name__) + + +class SelectField(object): + """ + A select field. + + >>> t = SelectField( "foo", multiple=True ) + >>> t.add_option( "tuti", 1 ) + >>> t.add_option( "fruity", "x" ) + >>> print(t.get_html()) + + + >>> t = SelectField( "bar" ) + >>> t.add_option( "automatic", 3 ) + >>> t.add_option( "bazooty", 4, selected=True ) + >>> print(t.get_html()) + + + >>> t = SelectField( "foo", display="radio" ) + >>> t.add_option( "tuti", 1 ) + >>> t.add_option( "fruity", "x" ) + >>> print(t.get_html()) +
+
+ + >>> t = SelectField( "bar", multiple=True, display="checkboxes" ) + >>> t.add_option( "automatic", 3 ) + >>> t.add_option( "bazooty", 4, selected=True ) + >>> print(t.get_html()) +
+
+
+ """ + + def __init__(self, name, multiple=None, display=None, refresh_on_change=False, refresh_on_change_values=None, size=None, field_id=None, value=None, selectlist=None, **kwds): + self.name = name + self.label = label + self.value = value + self.disabled = kwds.get('disabled', False) + if 'optional' in kwds: + self.optional = asbool(kwds.get('optional')) + else: + self.optional = kwds.get('required', 'optional') == 'optional' + self.help = kwds.get('helptext') + self.field_id = field_id + self.multiple = multiple or False + self.selectlist = selectlist or [] + self.size = size + self.options = list() + if display == "checkboxes": + assert multiple, "Checkbox display only supported for multiple select" + elif display == "radio": + assert not(multiple), "Radio display only supported for single select" + elif display is not None: + raise Exception("Unknown display type: %s" % display) + self.display = display + self.refresh_on_change = refresh_on_change + self.refresh_on_change_values = refresh_on_change_values or [] + if self.refresh_on_change: + self.refresh_on_change_text = ' refresh_on_change="true"' + if self.refresh_on_change_values: + self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % (self.refresh_on_change_text, escape(",".join(self.refresh_on_change_values), quote=True)) + else: + self.refresh_on_change_text = '' + + def get_disabled_str(self, disabled=False): + if disabled: + return ' disabled="disabled"' + else: + return '' + + def add_option(self, text, value, selected=False): + self.options.append((text, value, selected)) + + def get_html(self, prefix="", disabled=False, extra_attr=None): + if extra_attr is not None: + self.extra_attributes = ' %s' % ' '.join(['%s="%s"' % (k, escape(v)) for k, v in extra_attr.items()]) + else: + self.extra_attributes = '' + if self.display == "checkboxes": + return self.get_html_checkboxes(prefix, disabled) + elif self.display == "radio": + return self.get_html_radio(prefix, disabled) + else: + return self.get_html_default(prefix, disabled) + + def get_html_checkboxes(self, prefix="", disabled=False): + rval = [] + ctr = 0 + if len(self.options) > 1: + rval.append('
' % (prefix, self.name)) # placeholder for the insertion of the Select All/Unselect All buttons + for text, value, selected in self.options: + style = "" + text = unicodify(text) + escaped_value = escape(unicodify(value), quote=True) + uniq_id = "%s%s|%s" % (prefix, self.name, escaped_value) + if len(self.options) > 2 and ctr % 2 == 1: + style = " class=\"odd_row\"" + selected_text = "" + if selected: + selected_text = " checked='checked'" + rval.append('' + % (style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str(disabled), self.extra_attributes, uniq_id, escape(text, quote=True))) + ctr += 1 + return unicodify("\n".join(rval)) + + def get_html_radio(self, prefix="", disabled=False): + rval = [] + ctr = 0 + for text, value, selected in self.options: + style = "" + escaped_value = escape(str(value), quote=True) + uniq_id = "%s%s|%s" % (prefix, self.name, escaped_value) + if len(self.options) > 2 and ctr % 2 == 1: + style = " class=\"odd_row\"" + selected_text = "" + if selected: + selected_text = " checked='checked'" + rval.append('' + % (style, + prefix, + self.name, + self.refresh_on_change_text, + escaped_value, + uniq_id, + selected_text, + self.get_disabled_str(disabled), + self.extra_attributes, + uniq_id, + text)) + ctr += 1 + return unicodify("\n".join(rval)) + + def get_html_default(self, prefix="", disabled=False): + if self.multiple: + multiple = " multiple" + else: + multiple = "" + if self.size: + size = ' size="%s"' % str(self.size) + else: + size = '' + rval = [] + last_selected_value = "" + for text, value, selected in self.options: + if selected: + selected_text = " selected" + last_selected_value = value + if not isinstance(last_selected_value, string_types): + last_selected_value = str(last_selected_value) + else: + selected_text = "" + rval.append('' % (escape(unicodify(value), quote=True), selected_text, escape(unicodify(text), quote=True))) + if last_selected_value: + last_selected_value = ' last_selected_value="%s"' % escape(unicodify(last_selected_value), quote=True) + if self.field_id is not None: + id_string = ' id="%s"' % self.field_id + else: + id_string = '' + rval.insert(0, '') + return unicodify("\n".join(rval)) + + def get_selected(self, return_label=False, return_value=False, multi=False): + ''' + Return the currently selected option's label, value or both as a tuple. For + multi-select lists, a list is returned. + ''' + if multi: + selected_options = [] + for label, value, selected in self.options: + if selected: + if return_label and return_value: + if multi: + selected_options.append((label, value)) + else: + return (label, value) + elif return_label: + if multi: + selected_options.append(label) + else: + return label + elif return_value: + if multi: + selected_options.append(value) + else: + return value + if multi: + return selected_options + return None + + +def build_select_field(trans, objs, label_attr, select_field_name, initial_value='none', + selected_value='none', refresh_on_change=False, multiple=False, display=None, size=None): + """ + Build a SelectField given a set of objects. The received params are: + + - objs: the set of objects used to populate the option list + - label_attr: the attribute of each obj (e.g., name, email, etc ) whose value is used to populate each option label. + + - If the string 'self' is passed as label_attr, each obj in objs is assumed to be a string, so the obj itself is used + + - select_field_name: the name of the SelectField + - initial_value: the value of the first option in the SelectField - allows for an option telling the user to select something + - selected_value: the value of the currently selected option + - refresh_on_change: True if the SelectField should perform a refresh_on_change + """ + if initial_value == 'none': + values = [initial_value] + else: + values = [] + for obj in objs: + if label_attr == 'self': + # Each obj is a string + values.append(obj) + else: + values.append(trans.security.encode_id(obj.id)) + if refresh_on_change: + refresh_on_change_values = values + else: + refresh_on_change_values = [] + select_field = SelectField(name=select_field_name, + multiple=multiple, + display=display, + refresh_on_change=refresh_on_change, + refresh_on_change_values=refresh_on_change_values, + size=size) + for obj in objs: + if label_attr == 'self': + # Each obj is a string + if str(selected_value) == str(obj): + select_field.add_option(obj, obj, selected=True) + else: + select_field.add_option(obj, obj) + else: + label = getattr(obj, label_attr) + if str(selected_value) == str(obj.id) or str(selected_value) == trans.security.encode_id(obj.id): + select_field.add_option(label, trans.security.encode_id(obj.id), selected=True) + else: + select_field.add_option(label, trans.security.encode_id(obj.id)) + return select_field From 0dcad1a485e677de244f7483fb6210ebea47ad58 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 17:01:39 -0500 Subject: [PATCH 062/208] Import toolshed form builder in remaining modules --- .../installed_repository_metadata_manager.py | 2 +- .../metadata/repository_metadata_manager.py | 2 +- lib/tool_shed/repository_types/util.py | 2 +- lib/tool_shed/util/admin_util.py | 2 +- lib/tool_shed/util/form_util.py | 119 +++++++++++++++--- lib/tool_shed/util/tool_dependency_util.py | 2 +- lib/tool_shed/util/tool_util.py | 2 +- 7 files changed, 110 insertions(+), 21 deletions(-) diff --git a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py index fc2f50dc0a9..bc14cf28a4f 100644 --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -5,7 +5,7 @@ from sqlalchemy import false from galaxy import util from galaxy.util import inflector -from galaxy.web.form_builder import SelectField +from tool_shed.util.form_util import SelectField from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.metadata import metadata_generator from tool_shed.util import common_util diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py index fb8c60a4f31..179e53199f9 100644 --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -5,10 +5,10 @@ from sqlalchemy import false, or_ from galaxy import util from galaxy.util import inflector -from galaxy.web.form_builder import SelectField from tool_shed.metadata import metadata_generator from tool_shed.repository_types import util as rt_util from tool_shed.repository_types.metadata import TipOnly +from tool_shed.util.form_util import SelectField from tool_shed.util import (basic_util, common_util, hg_util, metadata_util, repository_util, shed_util_common as suc, tool_util) diff --git a/lib/tool_shed/repository_types/util.py b/lib/tool_shed/repository_types/util.py index 0e1e9447c2b..25b3ed3a4e9 100644 --- a/lib/tool_shed/repository_types/util.py +++ b/lib/tool_shed/repository_types/util.py @@ -1,6 +1,6 @@ import logging -from galaxy.web.form_builder import SelectField +from tool_shed.util.form_util import SelectField log = logging.getLogger(__name__) diff --git a/lib/tool_shed/util/admin_util.py b/lib/tool_shed/util/admin_util.py index c46f479e77c..b9fb8efe611 100644 --- a/lib/tool_shed/util/admin_util.py +++ b/lib/tool_shed/util/admin_util.py @@ -4,7 +4,7 @@ from sqlalchemy import false, func from galaxy import util, web from galaxy.util import inflector -from galaxy.web.form_builder import CheckboxField +from tool_shed.util.form_util import CheckboxField from tool_shed.util.web_util import escape diff --git a/lib/tool_shed/util/form_util.py b/lib/tool_shed/util/form_util.py index 631897cb91d..d0a0108d3cf 100644 --- a/lib/tool_shed/util/form_util.py +++ b/lib/tool_shed/util/form_util.py @@ -17,7 +17,90 @@ from galaxy.util import ( log = logging.getLogger(__name__) -class SelectField(object): +class BaseField(object): + def __init__(self, name, value=None, label=None, **kwds): + self.name = name + self.label = label + self.value = value + self.disabled = kwds.get('disabled', False) + if 'optional' in kwds: + self.optional = asbool(kwds.get('optional')) + else: + self.optional = kwds.get('required', 'optional') == 'optional' + self.help = kwds.get('helptext') + + def get_html(self, prefix=""): + """Returns the html widget corresponding to the parameter""" + raise TypeError("Abstract Method") + + def get_disabled_str(self, disabled=False): + if disabled: + return ' disabled="disabled"' + else: + return '' + + def to_dict(self): + return { + 'name' : self.name, + 'label' : self.label, + 'disabled' : self.disabled, + 'optional' : self.optional, + 'value' : self.value, + 'help' : self.help + } + + +class CheckboxField(BaseField): + """ + A checkbox (boolean input) + + >>> print(CheckboxField( "foo" ).get_html()) + + >>> print(CheckboxField( "bar", checked="yes" ).get_html()) + + """ + + def __init__(self, name, checked=None, refresh_on_change=False, refresh_on_change_values=None, value=None, **kwds): + super(CheckboxField, self).__init__(name, value, **kwds) + self.name = name + self.checked = (checked is True) or (isinstance(checked, string_types) and (checked.lower() in ("yes", "true", "on"))) + self.refresh_on_change = refresh_on_change + self.refresh_on_change_values = refresh_on_change_values or [] + if self.refresh_on_change: + self.refresh_on_change_text = ' refresh_on_change="true" ' + if self.refresh_on_change_values: + self.refresh_on_change_text = '%s refresh_on_change_values="%s" ' % (self.refresh_on_change_text, ",".join(self.refresh_on_change_values)) + else: + self.refresh_on_change_text = '' + + def get_html(self, prefix="", disabled=False): + if self.checked: + checked_text = ' checked="checked"' + else: + checked_text = '' + id_name = prefix + self.name + return unicodify('' + % (id_name, id_name, checked_text, self.get_disabled_str(disabled), self.refresh_on_change_text, id_name, self.get_disabled_str(disabled))) + + @staticmethod + def is_checked(value): + if value in [True, "true"]: + return True + return isinstance(value, list) and ('__CHECKED__' in value or len(value) == 2) + + def set_checked(self, value): + if isinstance(value, string_types): + self.checked = value.lower() in ["yes", "true", "on"] + else: + self.checked = value + + def to_dict(self): + d = super(CheckboxField, self).to_dict() + d['type'] = 'boolean' + return d + + +class SelectField(BaseField): """ A select field. @@ -56,18 +139,12 @@ class SelectField(object): """ def __init__(self, name, multiple=None, display=None, refresh_on_change=False, refresh_on_change_values=None, size=None, field_id=None, value=None, selectlist=None, **kwds): + super(SelectField, self).__init__(name, value, **kwds) self.name = name - self.label = label - self.value = value - self.disabled = kwds.get('disabled', False) - if 'optional' in kwds: - self.optional = asbool(kwds.get('optional')) - else: - self.optional = kwds.get('required', 'optional') == 'optional' - self.help = kwds.get('helptext') self.field_id = field_id self.multiple = multiple or False self.selectlist = selectlist or [] + self.value = value self.size = size self.options = list() if display == "checkboxes": @@ -86,12 +163,6 @@ class SelectField(object): else: self.refresh_on_change_text = '' - def get_disabled_str(self, disabled=False): - if disabled: - return ' disabled="disabled"' - else: - return '' - def add_option(self, text, value, selected=False): self.options.append((text, value, selected)) @@ -213,7 +284,25 @@ class SelectField(object): return selected_options return None + def to_dict(self): + d = super(SelectField, self).to_dict() + d['type'] = 'select' + d['display'] = self.display + d['multiple'] = self.multiple + d['data'] = [] + for value in self.selectlist: + d['data'].append({'label': value, 'value': value}) + return d + +def get_suite(): + """Get unittest suite for this module""" + import doctest + import sys + return doctest.DocTestSuite(sys.modules[__name__]) + + +# --------- Utility methods ----------------------------- def build_select_field(trans, objs, label_attr, select_field_name, initial_value='none', selected_value='none', refresh_on_change=False, multiple=False, display=None, size=None): """ diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index e470f9fffd8..f88d21ed951 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -5,7 +5,7 @@ import shutil from sqlalchemy import and_ from galaxy import util -from galaxy.web.form_builder import SelectField +from tool_shed.util.form_util import SelectField from tool_shed.util import hg_util from tool_shed.util import xml_util diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index da0518b1659..27eacdf4f4d 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -7,7 +7,7 @@ from galaxy import util from galaxy.datatypes.sniff import is_column_based from galaxy.util import checkers from galaxy.util.expressions import ExpressionContext -from galaxy.web.form_builder import SelectField +from tool_shed.util.form_util import SelectField from tool_shed.util import basic_util log = logging.getLogger(__name__) From c1acba4a5ba63c4ef0d6e4b69e7c1840d87cae28 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 17:03:23 -0500 Subject: [PATCH 063/208] Remove to dictionary helpers from toolshed form builder --- lib/tool_shed/util/form_util.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/lib/tool_shed/util/form_util.py b/lib/tool_shed/util/form_util.py index d0a0108d3cf..792ebdcba92 100644 --- a/lib/tool_shed/util/form_util.py +++ b/lib/tool_shed/util/form_util.py @@ -39,16 +39,6 @@ class BaseField(object): else: return '' - def to_dict(self): - return { - 'name' : self.name, - 'label' : self.label, - 'disabled' : self.disabled, - 'optional' : self.optional, - 'value' : self.value, - 'help' : self.help - } - class CheckboxField(BaseField): """ @@ -94,11 +84,6 @@ class CheckboxField(BaseField): else: self.checked = value - def to_dict(self): - d = super(CheckboxField, self).to_dict() - d['type'] = 'boolean' - return d - class SelectField(BaseField): """ @@ -284,16 +269,6 @@ class SelectField(BaseField): return selected_options return None - def to_dict(self): - d = super(SelectField, self).to_dict() - d['type'] = 'select' - d['display'] = self.display - d['multiple'] = self.multiple - d['data'] = [] - for value in self.selectlist: - d['data'].append({'label': value, 'value': value}) - return d - def get_suite(): """Get unittest suite for this module""" From 2c1a9a6b7a910fb571e7b6c268ba952706e92644 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 17:10:04 -0500 Subject: [PATCH 064/208] Use toolshed form builder for admin toolshed --- lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index faa6fd56576..06061e9e641 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -9,7 +9,7 @@ import tool_shed.repository_types.util as rt_util from galaxy import util, web from galaxy.tools.deps import views from galaxy.util import unicodify -from galaxy.web.form_builder import CheckboxField +from tool_shed.util.form_util import CheckboxField from tool_shed.galaxy_install import ( dependency_display, install_manager From 0b3bcb748548dbd7bd58c54b03c273912dca96e9 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 17:12:35 -0500 Subject: [PATCH 065/208] Avoid uninitialized variable in toolshed cloning process --- lib/tool_shed/util/hg_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index f979e73c8cc..30b3e5e4d02 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -54,6 +54,7 @@ def clone_repository(repository_clone_url, repository_file_dir, ctx_rev): Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository. """ + stdouterr = None try: stdouterr = subprocess.check_output(['hg', 'clone', '-r', ctx_rev, repository_clone_url, repository_file_dir], stderr=subprocess.STDOUT) return True, None From c2f92fdc4bcc9c440a97503f94090a748ef6c30a Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 17:29:58 -0500 Subject: [PATCH 066/208] Restore dict helper, some toolshed elements use the new client but the toolshed backend --- lib/tool_shed/util/form_util.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/tool_shed/util/form_util.py b/lib/tool_shed/util/form_util.py index 792ebdcba92..8bd4564a866 100644 --- a/lib/tool_shed/util/form_util.py +++ b/lib/tool_shed/util/form_util.py @@ -39,6 +39,15 @@ class BaseField(object): else: return '' + def to_dict(self): + return { + 'name' : self.name, + 'label' : self.label, + 'disabled' : self.disabled, + 'optional' : self.optional, + 'value' : self.value, + 'help' : self.help + } class CheckboxField(BaseField): """ @@ -84,6 +93,11 @@ class CheckboxField(BaseField): else: self.checked = value + def to_dict(self): + d = super(CheckboxField, self).to_dict() + d['type'] = 'boolean' + return d + class SelectField(BaseField): """ @@ -269,6 +283,16 @@ class SelectField(BaseField): return selected_options return None + def to_dict(self): + d = super(SelectField, self).to_dict() + d['type'] = 'select' + d['display'] = self.display + d['multiple'] = self.multiple + d['data'] = [] + for value in self.selectlist: + d['data'].append({'label': value, 'value': value}) + return d + def get_suite(): """Get unittest suite for this module""" From 87e44fbdfe1a84be77603b77043249ad73219ef6 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 1 Dec 2017 23:46:25 -0500 Subject: [PATCH 067/208] Fix import order --- lib/galaxy/model/__init__.py | 5 ++--- lib/galaxy/web/form_builder.py | 9 ++------- lib/galaxy/webapps/galaxy/api/library_contents.py | 4 ---- .../webapps/galaxy/controllers/admin_toolshed.py | 2 +- .../metadata/installed_repository_metadata_manager.py | 3 +-- lib/tool_shed/grids/util.py | 2 +- lib/tool_shed/metadata/repository_metadata_manager.py | 2 +- lib/tool_shed/util/form_util.py | 2 +- lib/tool_shed/util/tool_dependency_util.py | 10 ++++++---- lib/tool_shed/util/tool_util.py | 2 +- 10 files changed, 16 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a0ae115752d..d4abdb59c2a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,7 +5,6 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ import errno -import json import logging import numbers import operator @@ -30,8 +29,8 @@ from galaxy.managers import tags from galaxy.model.item_attrs import UsesAnnotations from galaxy.model.util import pgcalc from galaxy.security import get_permitted_actions -from galaxy.util import (directory_hash_id, Params, ready_name_for_url, - restore_text, unicodify, unique_id) +from galaxy.util import (directory_hash_id, ready_name_for_url, + unicodify, unique_id) from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable from galaxy.util.hash_util import new_secure_hash diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index f7656e11fae..ff177ccfb3a 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -4,14 +4,9 @@ Classes for generating HTML forms from __future__ import print_function import logging -from cgi import escape - -from six import string_types from galaxy.util import ( - asbool, - restore_text, - unicodify + asbool ) log = logging.getLogger(__name__) @@ -167,7 +162,7 @@ class WorkflowField(BaseField): class WorkflowMappingField(BaseField): - def __init__(self, name, user=None, value=None, **kwd): + def __init__(self, name, user=None, value=None, **kwds): super(WorkflowMappingField, self).__init__(name, value, **kwds) self.user = user diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index 551575b4a8e..befe4263a56 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -32,10 +32,6 @@ from galaxy.web.base.controller import ( UsesLibraryMixin, UsesLibraryMixinItems ) -from galaxy.web.form_builder import ( - AddressField, - CheckboxField, -) log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 06061e9e641..5758532a68e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -9,7 +9,6 @@ import tool_shed.repository_types.util as rt_util from galaxy import util, web from galaxy.tools.deps import views from galaxy.util import unicodify -from tool_shed.util.form_util import CheckboxField from tool_shed.galaxy_install import ( dependency_display, install_manager @@ -33,6 +32,7 @@ from tool_shed.util import ( tool_util, workflow_util ) +from tool_shed.util.form_util import CheckboxField from tool_shed.util.web_util import escape from .admin import AdminGalaxy diff --git a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py index bc14cf28a4f..821fae2d245 100644 --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -5,14 +5,13 @@ from sqlalchemy import false from galaxy import util from galaxy.util import inflector -from tool_shed.util.form_util import SelectField from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.metadata import metadata_generator from tool_shed.util import common_util from tool_shed.util import repository_util from tool_shed.util import tool_util from tool_shed.util import xml_util - +from tool_shed.util.form_util import SelectField log = logging.getLogger(__name__) diff --git a/lib/tool_shed/grids/util.py b/lib/tool_shed/grids/util.py index 72536e2c7a1..4df8079d5da 100644 --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,7 +1,7 @@ import logging -from tool_shed.util.form_util import SelectField from tool_shed.util import hg_util, metadata_util +from tool_shed.util.form_util import SelectField log = logging.getLogger(__name__) diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py index 179e53199f9..9379adecb63 100644 --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -8,9 +8,9 @@ from galaxy.util import inflector from tool_shed.metadata import metadata_generator from tool_shed.repository_types import util as rt_util from tool_shed.repository_types.metadata import TipOnly -from tool_shed.util.form_util import SelectField from tool_shed.util import (basic_util, common_util, hg_util, metadata_util, repository_util, shed_util_common as suc, tool_util) +from tool_shed.util.form_util import SelectField log = logging.getLogger(__name__) diff --git a/lib/tool_shed/util/form_util.py b/lib/tool_shed/util/form_util.py index 8bd4564a866..fd874ab36b6 100644 --- a/lib/tool_shed/util/form_util.py +++ b/lib/tool_shed/util/form_util.py @@ -10,7 +10,6 @@ from six import string_types from galaxy.util import ( asbool, - restore_text, unicodify ) @@ -49,6 +48,7 @@ class BaseField(object): 'help' : self.help } + class CheckboxField(BaseField): """ A checkbox (boolean input) diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index f88d21ed951..a3654a88712 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -5,9 +5,11 @@ import shutil from sqlalchemy import and_ from galaxy import util -from tool_shed.util.form_util import SelectField -from tool_shed.util import hg_util -from tool_shed.util import xml_util +from tool_shed.util import ( + form_util, + hg_util, + xml_util +) log = logging.getLogger(__name__) @@ -18,7 +20,7 @@ def build_tool_dependencies_select_field(app, tool_shed_repository, name, multip Generate a SelectField consisting of the current list of tool dependency ids for an installed tool shed repository. """ - tool_dependencies_select_field = SelectField(name=name, multiple=multiple, display=display) + tool_dependencies_select_field = form_util.SelectField(name=name, multiple=multiple, display=display) for tool_dependency in tool_shed_repository.tool_dependencies: if uninstalled_only: if tool_dependency.status not in [app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 27eacdf4f4d..1e374f655f2 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -7,8 +7,8 @@ from galaxy import util from galaxy.datatypes.sniff import is_column_based from galaxy.util import checkers from galaxy.util.expressions import ExpressionContext -from tool_shed.util.form_util import SelectField from tool_shed.util import basic_util +from tool_shed.util.form_util import SelectField log = logging.getLogger(__name__) From 644c108e28e3c804e2f9841d5d4ac377157be3fe Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 2 Dec 2017 10:19:56 -0500 Subject: [PATCH 068/208] Remove formbuilder from framework import --- lib/galaxy/web/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/galaxy/web/__init__.py b/lib/galaxy/web/__init__.py index a522f38e040..a7284fffafe 100644 --- a/lib/galaxy/web/__init__.py +++ b/lib/galaxy/web/__init__.py @@ -23,10 +23,6 @@ from .framework.decorators import ( require_admin, require_login ) -from .framework.formbuilder import ( - form, - FormBuilder -) __all__ = ('url_for', 'error', 'expose', 'json', 'json_pretty', 'require_admin', 'require_login', 'expose_api', 'expose_api_anonymous', From 141a321bfe3ee35cd1a199f42f5e55d131775c0d Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 4 Dec 2017 12:31:14 -0500 Subject: [PATCH 069/208] Remove unused panel section parsing --- lib/galaxy/webapps/galaxy/api/toolshed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/toolshed.py b/lib/galaxy/webapps/galaxy/api/toolshed.py index a01950f91fd..05fd3c50073 100644 --- a/lib/galaxy/webapps/galaxy/api/toolshed.py +++ b/lib/galaxy/webapps/galaxy/api/toolshed.py @@ -252,7 +252,6 @@ class ToolShedController(BaseAPIController): repository_data['repository'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', repository_id])) repository_data['repository']['metadata'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', repository_id, 'metadata'])) repository_data['shed_conf'] = tool_util.build_shed_tool_conf_select_field(trans.app).to_dict() - repository_data['panel_section'] = tool_panel_section_select_field.to_dict() repository_data['panel_section_dict'] = tool_panel_section_dict for changeset, metadata in repository_data['repository']['metadata'].items(): if changeset not in tool_dependencies: From da9d9bf61cf90dd6d3b12450bf1cdceebd2d64ea Mon Sep 17 00:00:00 2001 From: guerler Date: Mon, 4 Dec 2017 12:35:46 -0500 Subject: [PATCH 070/208] Remove panel section from duplicate toolshed api endpoint --- lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 8871b893e1f..60f6e4833af 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -264,7 +264,6 @@ class ToolShedRepositoriesController(BaseAPIController): repository_data['repository'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', tsr_id])) repository_data['repository']['metadata'] = json.loads(util.url_get(tool_shed_url, pathspec=['api', 'repositories', tsr_id, 'metadata'])) repository_data['shed_conf'] = tool_util.build_shed_tool_conf_select_field(trans.app).to_dict() - repository_data['panel_section'] = tool_panel_section_select_field.to_dict() repository_data['panel_section_dict'] = tool_panel_section_dict for changeset, metadata in repository_data['repository']['metadata'].items(): if changeset not in tool_dependencies: From c7ebf747f9fef827e89893776ee77870b996d70a Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 4 Dec 2017 15:40:25 -0500 Subject: [PATCH 071/208] Fix the shed tool conf select field. --- client/galaxy/scripts/mvc/toolshed/repository-view.js | 9 +++++++-- lib/tool_shed/util/form_util.py | 5 ++--- static/scripts/bundled/adminToolshed.bundled.js | 2 +- static/scripts/mvc/toolshed/repository-view.js | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/client/galaxy/scripts/mvc/toolshed/repository-view.js b/client/galaxy/scripts/mvc/toolshed/repository-view.js index 50dab10c3d9..cac8bded072 100644 --- a/client/galaxy/scripts/mvc/toolshed/repository-view.js +++ b/client/galaxy/scripts/mvc/toolshed/repository-view.js @@ -40,8 +40,9 @@ var ToolShedRepositoryView = Backbone.View.extend({ this.options.tps_select_options = this.templatePanelSelectOptions; this.options.tool_dependencies = models.get("tool_dependencies"); this.options.shed_tool_conf = this.templateShedToolConf({ - stc_html: models.get("shed_conf") + shed_tool_confs: models.get("shed_conf") }); + console.log(models.get("shed_conf")); this.options.panel_section_dict = models.get("panel_section_dict"); this.options.api_url = `${Galaxy.root}api/tool_shed_repositories/install?async=True`; this.options = _.extend(this.options, options); @@ -467,7 +468,11 @@ var ToolShedRepositoryView = Backbone.View.extend({ '
Shed tool configuration file:
', '
', '
', - "<%= stc_html %>", + '', '
Select the file whose tool_path setting you want used for installing repositories.
', "
", "
" diff --git a/lib/tool_shed/util/form_util.py b/lib/tool_shed/util/form_util.py index fd874ab36b6..ea1d1527f0b 100644 --- a/lib/tool_shed/util/form_util.py +++ b/lib/tool_shed/util/form_util.py @@ -288,9 +288,8 @@ class SelectField(BaseField): d['type'] = 'select' d['display'] = self.display d['multiple'] = self.multiple - d['data'] = [] - for value in self.selectlist: - d['data'].append({'label': value, 'value': value}) + d['data'] = [{'label': value, 'value': value} for value in self.selectlist] + d['options'] = [{'label': option[0], 'value': option[1], 'selected': option[2]} for option in self.options] return d diff --git a/static/scripts/bundled/adminToolshed.bundled.js b/static/scripts/bundled/adminToolshed.bundled.js index 0732a93d49c..ed1c9ee23ae 100644 --- a/static/scripts/bundled/adminToolshed.bundled.js +++ b/static/scripts/bundled/adminToolshed.bundled.js @@ -46,4 +46,4 @@ r=i,l=e("
",{css:{position:"absolute",top:"-200px",left:s?"0px":"-1000px", // the check will work anyway, as when moving the node will be loaded first and checked again if(v === 'i' && !ins.is_loaded(p)) { } */ -p=!0,_=0,f=o.data.nodes.length;_e.inArray(o.data.nodes[_],v.children)&&(m-=1)),!(p=p&&(w&&w.settings&&w.settings.dnd&&!1===w.settings.dnd.check_while_dragging||w.check(g,o.data.origin&&o.data.origin!==w?o.data.origin.get_node(o.data.nodes[_]):o.data.nodes[_],c,m,{dnd:!0,ref:w.get_node(x.parent()),pos:j,is_multi:o.data.origin&&o.data.origin!==w,is_foreign:!o.data.origin})))){w&&w.last_error&&(i=w.last_error());break}if(p)return"i"===j&&x.parent().is(".jstree-closed")&&w.settings.dnd.open_timeout&&(s=setTimeout(function(e,t){return function(){e.open_node(t)}}(w,x),w.settings.dnd.open_timeout)),t={ins:w,par:c,pos:"i"!==j||"last"!==b||0!==h||w.is_loaded(y)?h:"last"},n.css({left:r+"px",top:l+"px"}).show(),o.helper.find(".jstree-icon:eq(0)").removeClass("jstree-er").addClass("jstree-ok"),i={},u=!0,!1}),!0===u))return}else{for(p=!0,_=0,f=o.data.nodes.length;_
").html(n).css({display:"block",margin:"0",padding:"0",position:"absolute",top:"-2000px",lineHeight:"16px",zIndex:"10000"})),e(document).bind("mousemove touchmove",e.vakata.dnd.drag),e(document).bind("mouseup touchend",e.vakata.dnd.stop),!1},drag:function(i){if("touchmove"===i.type&&i.originalEvent&&i.originalEvent.changedTouches&&i.originalEvent.changedTouches[0]&&(i.pageX=i.originalEvent.changedTouches[0].pageX,i.pageY=i.originalEvent.changedTouches[0].pageY,i.target=document.elementFromPoint(i.originalEvent.changedTouches[0].pageX-window.pageXOffset,i.originalEvent.changedTouches[0].pageY-window.pageYOffset)),t.is_down){if(!t.is_drag){if(!(Math.abs(i.pageX-t.init_x)>(t.is_touch?e.vakata.dnd.settings.threshold_touch:e.vakata.dnd.settings.threshold)||Math.abs(i.pageY-t.init_y)>(t.is_touch?e.vakata.dnd.settings.threshold_touch:e.vakata.dnd.settings.threshold)))return;t.helper&&(t.helper.appendTo("body"),t.helper_w=t.helper.outerWidth()),t.is_drag=!0,e.vakata.dnd._trigger("start",i)}var s=!1,n=!1,a=!1,o=!1,r=!1,l=!1,d=!1,c=!1,h=!1,u=!1;return t.scroll_t=0,t.scroll_l=0,t.scroll_e=!1,e(e(i.target).parentsUntil("body").addBack().get().reverse()).filter(function(){return/^auto|scroll$/.test(e(this).css("overflow"))&&(this.scrollHeight>this.offsetHeight||this.scrollWidth>this.offsetWidth)}).each(function(){var s=e(this),n=s.offset();if(this.scrollHeight>this.offsetHeight&&(n.top+s.height()-i.pageYthis.offsetWidth&&(n.left+s.width()-i.pageXo&&i.pageY-do&&o-(i.pageY-d)l&&i.pageX-cl&&l-(i.pageX-c)a&&(h=a-50),r&&u+t.helper_w>r&&(u=r-(t.helper_w+2)),t.helper.css({left:u+"px",top:h+"px"})),e.vakata.dnd._trigger("move",i),!1}},stop:function(i){if("touchend"===i.type&&i.originalEvent&&i.originalEvent.changedTouches&&i.originalEvent.changedTouches[0]&&(i.pageX=i.originalEvent.changedTouches[0].pageX,i.pageY=i.originalEvent.changedTouches[0].pageY,i.target=document.elementFromPoint(i.originalEvent.changedTouches[0].pageX-window.pageXOffset,i.originalEvent.changedTouches[0].pageY-window.pageYOffset)),t.is_drag)e.vakata.dnd._trigger("stop",i);else if("touchend"===i.type&&i.target===t.target){var s=setTimeout(function(){e(i.target).click()},100);e(i.target).one("click",function(){s&&clearTimeout(s)})}return e.vakata.dnd._clean(),!1}}}(e),e.jstree.defaults.search={ajax:!1,fuzzy:!1,case_sensitive:!1,show_only_matches:!1,close_opened_onclear:!0,search_leaves_only:!1,search_callback:!1},e.jstree.plugins.search=function(t,i){this.bind=function(){i.bind.call(this),this._data.search.str="",this._data.search.dom=e(),this._data.search.res=[],this._data.search.opn=[],this.element.on("before_open.jstree",e.proxy(function(t,i){var s,n,a=this._data.search.res,o=[],r=e();if(a&&a.length&&(this._data.search.dom=e(this.element[0].querySelectorAll("#"+e.map(a,function(t){return-1!=="0123456789".indexOf(t[0])?"\\3"+t[0]+" "+t.substr(1).replace(e.jstree.idregex,"\\$&"):t.replace(e.jstree.idregex,"\\$&")}).join(", #"))),this._data.search.dom.children(".jstree-anchor").addClass("jstree-search"),this.settings.search.show_only_matches&&this._data.search.res.length)){for(s=0,n=a.length;s32&&(i.fuzzy=!1),i.fuzzy&&(s=1<=_;l--)if(m=n[t.charAt(l-1)],g[l]=0===o?(g[l+1]<<1|1)&m:(g[l+1]<<1|1)&m|(p[l+1]|p[l])<<1|1|p[l+1],g[l]&s&&(x=a(o,l-1))<=b){if(b=x,y=l-1,k.push(y),!(y>r))break;_=Math.max(1,2*r-y)}if(a(o+1,r)>b)break;p=g}return{isMatch:y>=0,score:x}},!0===t?{search:o}:o(t)}}(e),e.jstree.defaults.sort=function(e,t){return this.get_text(e)>this.get_text(t)?1:-1},e.jstree.plugins.sort=function(t,i){this.bind=function(){i.bind.call(this),this.element.on("model.jstree",e.proxy(function(e,t){this.sort(t.parent,!0)},this)).on("rename_node.jstree create_node.jstree",e.proxy(function(e,t){this.sort(t.parent||t.node.parent,!1),this.redraw_node(t.parent||t.node.parent,!0)},this)).on("move_node.jstree copy_node.jstree",e.proxy(function(e,t){this.sort(t.parent,!1),this.redraw_node(t.parent,!0)},this))},this.sort=function(t,i){var s,n;if((t=this.get_node(t))&&t.children&&t.children.length&&(t.children.sort(e.proxy(this.settings.sort,this)),i))for(s=0,n=t.children_d.length;st.ttl)&&(t&&t.state&&(t=t.state),t&&e.isFunction(this.settings.state.filter)&&(t=this.settings.state.filter.call(this,t)),!!t&&(this.element.one("set_state.jstree",function(i,s){s.instance.trigger("restore_state",{state:e.extend(!0,{},t)})}),this.set_state(t),!0))},this.clear_state=function(){return e.vakata.storage.del(this.settings.state.key)}},function(e,t){e.vakata.storage={set:function(e,t){return window.localStorage.setItem(e,t)},get:function(e){return window.localStorage.getItem(e)},del:function(e){return window.localStorage.removeItem(e)}}}(e),e.jstree.defaults.types={"#":{},default:{}},e.jstree.plugins.types=function(i,s){this.init=function(e,i){var n,a;if(i&&i.types&&i.types.default)for(n in i.types)if("default"!==n&&"#"!==n&&i.types.hasOwnProperty(n))for(a in i.types.default)i.types.default.hasOwnProperty(a)&&i.types[n][a]===t&&(i.types[n][a]=i.types.default[a]);s.init.call(this,e,i),this._model.data["#"].type="#"},this.refresh=function(e,t){s.refresh.call(this,e,t),this._model.data["#"].type="#"},this.bind=function(){this.element.on("model.jstree",e.proxy(function(e,i){var s,n,a=this._model.data,o=i.nodes,r=this.settings.types,l="default";for(s=0,n=o.length;s .jstree-ocl",e.proxy(function(t){t.stopImmediatePropagation();var i=e.Event("click",{metaKey:t.metaKey,ctrlKey:t.ctrlKey,altKey:t.altKey,shiftKey:t.shiftKey});e(t.currentTarget).closest(".jstree-node").children(".jstree-anchor:eq(0)").trigger(i).focus()},this)).on("mouseover.jstree",".jstree-wholerow, .jstree-icon",e.proxy(function(e){return e.stopImmediatePropagation(),this.hover_node(e.currentTarget),!1},this)).on("mouseleave.jstree",".jstree-node",e.proxy(function(e){this.dehover_node(e.currentTarget)},this))},this.teardown=function(){this.settings.wholerow&&this.element.find(".jstree-wholerow").remove(),i.teardown.call(this)},this.redraw_node=function(t,s,n){if(t=i.redraw_node.call(this,t,s,n)){var a=_.cloneNode(!0);-1!==e.inArray(t.id,this._data.core.selected)&&(a.className+=" jstree-wholerow-clicked"),t.insertBefore(a,t.childNodes[0])}return t}}}})},20:function(e,t,i){"use strict";(function(e,s,n){function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=i(3),r=(a(o),i(11)),l=a(r),d=i(4),c=a(d),h=i(48),u=a(h),p=i(44),_=a(p);t.default=e.View.extend({initialize:function(t){this.model=new e.Model({initial_errors:!1,cls:"ui-portlet-limited",icon:null,always_refresh:!0,status:"warning",hide_operations:!1,onchange:function(){}}).set(t),this.setElement("
"),this.render()},update:function(e){var t=this;this.data.matchModel(e,function(e,i){var n=t.input_list[i];if(n&&n.options&&!s.isEqual(n.options,e.options)){n.options=e.options;var a=t.field_list[i];if(a.update){var o=[];if(-1!=["data","data_collection","drill_down"].indexOf(n.type))o=n.options;else for(var r in e.options){var l=e.options[r];l.length>2&&o.push({label:l[0],value:l[1]})}a.update(o),a.trigger("change"),Galaxy.emit.debug("form-view::update()","Updating options for "+i)}}})},wait:function(e){for(var t in this.input_list){var i=this.field_list[t];this.input_list[t].is_dynamic&&i.wait&&i.unwait&&i[e?"wait":"unwait"]()}},highlight:function(e,t,i){var s=this.element_list[e];if(s&&(s.error(t||"Please verify this parameter."),this.portlet.expand(),this.trigger("expand",e),!i)){var a=this.$el.parents().filter(function(){return-1!=["auto","scroll"].indexOf(n(this).css("overflow"))}).first();a.animate({scrollTop:a.scrollTop()+s.$el.offset().top-120},500)}},errors:function(e){if(this.trigger("reset"),e&&e.errors){var t=this.data.matchResponse(e.errors);for(var i in this.element_list){this.element_list[i];t[i]&&this.highlight(i,t[i],!0)}}},render:function(){var e=this;this.off("change"),this.off("reset"),this.field_list={},this.input_list={},this.element_list={},this.data=new _.default.Manager(this),this._renderForm(),this.data.create(),this.model.get("initial_errors")&&this.errors(this.model.attributes);var t=this.data.checksum();return this.on("change",function(i){var s=e.input_list[i];if(!s||s.refresh_on_change||e.model.get("always_refresh")){var n=e.data.checksum();n!=t&&(t=n,e.model.get("onchange")())}}),this.on("reset",function(){s.each(e.element_list,function(e){e.reset()})}),this},_renderForm:function(){n(".tooltip").remove();var e=this.model.attributes;this.message=new c.default.UnescapedMessage,this.section=new u.default.View(this,{inputs:e.inputs}),this.portlet=new l.default.View({icon:e.icon,title:e.title,cls:e.cls,operations:!e.hide_operations&&e.operations,buttons:e.buttons,collapsible:e.collapsible,collapsed:e.collapsed,onchange_title:e.onchange_title}),this.portlet.append(this.message.$el),this.portlet.append(this.section.$el),this.$el.empty(),e.inputs&&this.$el.append(this.portlet.$el),e.message&&this.message.update({persistent:!0,status:e.status,message:e.message}),Galaxy.emit.debug("form-view::initialize()","Completed")}})}).call(t,i(1),i(2),i(0))},22:function(e,t,i){"use strict";(function(e,s,n){function a(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=i(3),r=a(o),l=i(12),d=a(l),c=e.View.extend({initialize:function(t){var i=this;this.data=[],this.data2=[],this.model=t&&t.model||new e.Model({id:r.default.uid(),cls:"ui-select",error_text:"No options available",empty_text:"Nothing selected",visible:!0,wait:!1,multiple:!1,searchable:!0,optional:!1,disabled:!1,readonly:!1,onchange:function(){},value:null,individual:!1,pagesize:20}).set(t),this.on("change",function(){i.model.get("onchange")&&i.model.get("onchange")(i.value())}),this.listenTo(this.model,"change:data",this._changeData,this),this.listenTo(this.model,"change:disabled",this._changeDisabled,this),this.listenTo(this.model,"change:wait",this._changeWait,this),this.listenTo(this.model,"change:visible",this._changeVisible,this),this.listenTo(this.model,"change:value",this._changeValue,this),this.listenTo(this.model,"change:multiple change:searchable change:cls change:id",this.render,this),this.render()},render:function(){var e=this;this.model.get("searchable")?this._renderSearchable():this._renderClassic(),this.$el.addClass(this.model.get("cls")).attr("id",this.model.get("id")),this.$select.empty().addClass("select").attr("id",this.model.get("id")+"_select").prop("multiple",this.model.get("multiple")).on("change",function(){e.value(e._getValue()),e.trigger("change")}),this._changeData(),this._changeWait(),this._changeVisible(),this._changeDisabled()},_renderClassic:function(){var e=this;this.$el.addClass(this.model.get("multiple")?"ui-select-multiple":"ui-select").append(this.$select=s("").attr({id:t,type:this.model.get("type"),name:this.model.id,value:e.value})).append(s("