From 12ce4e7266b2fb5985e8384eda62ff764930e60e Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:15:45 -0500 Subject: [PATCH 01/56] Strip legacy request helpers --- lib/galaxy/security/__init__.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index be1a576a112..32c2d1481da 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -1450,37 +1450,6 @@ class GalaxyRBACAgent(RBACAgent): hidden_folder_ids = '%d' % sub_folder.id return False, hidden_folder_ids - def can_access_request_type(self, roles, request_type): - action = self.permitted_actions.REQUEST_TYPE_ACCESS - request_type_actions = [] - for permission in request_type.actions: - if permission.action == action.action: - request_type_actions.append(permission) - if not request_type_actions: - return True - ret_val = False - for request_type_action in request_type_actions: - if request_type_action.role in roles: - ret_val = True - break - return ret_val - - def set_request_type_permissions(self, request_type, permissions={}): - # Set new permissions on request_type, eliminating all current permissions - for role_assoc in request_type.actions: - self.sa_session.delete(role_assoc) - # Add the new permissions on request_type - permission_class = self.model.RequestTypePermissions - flush_needed = False - for action, roles in permissions.items(): - if isinstance(action, Action): - action = action.action - for role_assoc in [permission_class(action, request_type, role) for role in roles]: - self.sa_session.add(role_assoc) - flush_needed = True - if flush_needed: - self.sa_session.flush() - class HostAgent(RBACAgent): """ From 67cf6944894f71a1408e0b596558039d00306f68 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:17:40 -0500 Subject: [PATCH 02/56] Remove field value helper --- lib/galaxy/web/base/controller.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 9cc3a9564a7..78b7478990f 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1289,21 +1289,6 @@ class UsesFormDefinitionsMixin: else: return [fdc.latest_form for fdc in fdc_list if fdc.latest_form.type == form_type] - def field_param_values_ok(self, widget_name, widget_type, **kwd): - # Make sure required fields have contents, etc - params = util.Params(kwd) - if widget_type == 'AddressField': - if not util.restore_text(params.get('%s_short_desc' % widget_name, '')) \ - or not util.restore_text(params.get('%s_name' % widget_name, '')) \ - or not util.restore_text(params.get('%s_institution' % widget_name, '')) \ - or not util.restore_text(params.get('%s_address' % widget_name, '')) \ - or not util.restore_text(params.get('%s_city' % widget_name, '')) \ - or not util.restore_text(params.get('%s_state' % widget_name, '')) \ - or not util.restore_text(params.get('%s_postal_code' % widget_name, '')) \ - or not util.restore_text(params.get('%s_country' % widget_name, '')): - return False - return True - def save_widget_field(self, trans, field_obj, widget_name, **kwd): # Save a form_builder field object params = util.Params(kwd) From de5d29a6f2e66bf2792c4cde0a7eb2a56e018234 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:19:51 -0500 Subject: [PATCH 03/56] Move grid helper to reports --- lib/galaxy/webapps/reports/framework/grids.py | 991 ++++++++++++++++++ 1 file changed, 991 insertions(+) create mode 100644 lib/galaxy/webapps/reports/framework/grids.py diff --git a/lib/galaxy/webapps/reports/framework/grids.py b/lib/galaxy/webapps/reports/framework/grids.py new file mode 100644 index 00000000000..fbfc11faf38 --- /dev/null +++ b/lib/galaxy/webapps/reports/framework/grids.py @@ -0,0 +1,991 @@ +import logging +import math +from json import dumps, loads + +from markupsafe import escape +from six import string_types, text_type +from sqlalchemy.sql.expression import and_, false, func, null, or_, true + +from galaxy.model.item_attrs import RuntimeException, UsesAnnotations, UsesItemRatings +from galaxy.util import restore_text, sanitize_text, unicodify +from galaxy.util.odict import odict +from galaxy.web.framework import decorators, url_for +from galaxy.web.framework.helpers import iff + + +log = logging.getLogger(__name__) + + +class Grid(object): + """ + Specifies the content and format of a grid (data table). + """ + title = "" + exposed = True + model_class = None + show_item_checkboxes = False + template = "grid_base.mako" + async_template = "grid_base_async.mako" + use_async = False + use_hide_message = True + global_actions = [] + columns = [] + operations = [] + standard_filters = [] + # Any columns that are filterable (either standard or advanced) should have a default value set in the default filter. + default_filter = {} + default_sort_key = None + preserve_state = False + use_paging = False + num_rows_per_page = 25 + num_page_links = 10 + # Set preference names. + cur_filter_pref_name = ".filter" + cur_sort_key_pref_name = ".sort_key" + pass_through_operations = {} + legend = None + info_text = None + + def __init__(self): + # Determine if any multiple row operations are defined + self.has_multiple_item_operations = False + for operation in self.operations: + if operation.allow_multiple: + self.has_multiple_item_operations = True + break + + # If a column does not have a model class, set the column's model class + # to be the grid's model class. + for column in self.columns: + if not column.model_class: + column.model_class = self.model_class + + def __call__(self, trans, **kwargs): + # Get basics. + # FIXME: pretty sure this is only here to pass along, can likely be eliminated + status = kwargs.get('status', None) + message = kwargs.get('message', None) + dict_format = kwargs.get('dict_format', False) + # Build a base filter and sort key that is the combination of the saved state and defaults. + # Saved state takes preference over defaults. + base_filter = {} + if self.default_filter: + # default_filter is a dictionary that provides a default set of filters based on the grid's columns. + base_filter = self.default_filter.copy() + base_sort_key = self.default_sort_key + if self.preserve_state: + pref_name = text_type(self.__class__.__name__ + self.cur_filter_pref_name) + if pref_name in trans.get_user().preferences: + saved_filter = loads(trans.get_user().preferences[pref_name]) + base_filter.update(saved_filter) + pref_name = text_type(self.__class__.__name__ + self.cur_sort_key_pref_name) + if pref_name in trans.get_user().preferences: + base_sort_key = loads(trans.get_user().preferences[pref_name]) + # Build initial query + query = self.build_initial_query(trans, **kwargs) + query = self.apply_query_filter(trans, query, **kwargs) + # Maintain sort state in generated urls + extra_url_args = {} + # Determine whether use_default_filter flag is set. + use_default_filter_str = kwargs.get('use_default_filter') + use_default_filter = False + if use_default_filter_str: + use_default_filter = (use_default_filter_str.lower() == 'true') + # Process filtering arguments to (a) build a query that represents the filter and (b) build a + # dictionary that denotes the current filter. + cur_filter_dict = {} + for column in self.columns: + if column.key: + # Get the filter criterion for the column. Precedence is (a) if using default filter, only look there; otherwise, + # (b) look in kwargs; and (c) look in base filter. + column_filter = None + if use_default_filter: + if self.default_filter: + column_filter = self.default_filter.get(column.key) + elif "f-" + column.model_class.__name__ + ".%s" % column.key in kwargs: + # Queries that include table joins cannot guarantee unique column names. This problem is + # handled by setting the column_filter value to .. + column_filter = kwargs.get("f-" + column.model_class.__name__ + ".%s" % column.key) + elif "f-" + column.key in kwargs: + column_filter = kwargs.get("f-" + column.key) + elif column.key in base_filter: + column_filter = base_filter.get(column.key) + + # Method (1) combines a mix of strings and lists of strings into a single string and (2) attempts to de-jsonify all strings. + def loads_recurse(item): + decoded_list = [] + if isinstance(item, string_types): + try: + # Not clear what we're decoding, so recurse to ensure that we catch everything. + decoded_item = loads(item) + if isinstance(decoded_item, list): + decoded_list = loads_recurse(decoded_item) + else: + decoded_list = [text_type(decoded_item)] + except ValueError: + decoded_list = [text_type(item)] + elif isinstance(item, list): + for element in item: + a_list = loads_recurse(element) + decoded_list = decoded_list + a_list + return decoded_list + # If column filter found, apply it. + if column_filter is not None: + # TextColumns may have a mix of json and strings. + if isinstance(column, TextColumn): + column_filter = loads_recurse(column_filter) + if len(column_filter) == 1: + column_filter = column_filter[0] + # Interpret ',' as a separator for multiple terms. + if isinstance(column_filter, string_types) and column_filter.find(',') != -1: + column_filter = column_filter.split(',') + + # Check if filter is empty + if isinstance(column_filter, list): + # Remove empty strings from filter list + column_filter = [x for x in column_filter if x != ''] + if len(column_filter) == 0: + continue + elif isinstance(column_filter, string_types): + # If filter criterion is empty, do nothing. + if column_filter == '': + continue + + # Update query. + query = column.filter(trans, trans.user, query, column_filter) + # Upate current filter dict. + # Column filters are rendered in various places, sanitize them all here. + cur_filter_dict[column.key] = sanitize_text(column_filter) + # Carry filter along to newly generated urls; make sure filter is a string so + # that we can encode to UTF-8 and thus handle user input to filters. + if isinstance(column_filter, list): + # Filter is a list; process each item. + column_filter = [text_type(_).encode('utf-8') if not isinstance(_, string_types) else _ for _ in column_filter] + extra_url_args["f-" + column.key] = dumps(column_filter) + else: + # Process singleton filter. + if not isinstance(column_filter, string_types): + column_filter = text_type(column_filter) + extra_url_args["f-" + column.key] = column_filter.encode("utf-8") + # Process sort arguments. + sort_key = None + if 'sort' in kwargs: + sort_key = kwargs['sort'] + elif base_sort_key: + sort_key = base_sort_key + if sort_key: + ascending = not(sort_key.startswith("-")) + # Queries that include table joins cannot guarantee unique column names. This problem is + # handled by setting the column_filter value to .. + table_name = None + if sort_key.find('.') > -1: + a_list = sort_key.split('.') + if ascending: + table_name = a_list[0] + else: + table_name = a_list[0][1:] + column_name = a_list[1] + elif ascending: + column_name = sort_key + else: + column_name = sort_key[1:] + # Sort key is a column key. + for column in self.columns: + if column.key and column.key.find('.') > -1: + column_key = column.key.split('.')[1] + else: + column_key = column.key + if (table_name is None or table_name == column.model_class.__name__) and column_key == column_name: + query = column.sort(trans, query, ascending, column_name=column_name) + break + extra_url_args['sort'] = sort_key + # There might be a current row + current_item = self.get_current_item(trans, **kwargs) + # Process page number. + if self.use_paging: + if 'page' in kwargs: + if kwargs['page'] == 'all': + page_num = 0 + else: + page_num = int(kwargs['page']) + else: + page_num = 1 + if page_num == 0: + # Show all rows in page. + total_num_rows = query.count() + page_num = 1 + num_pages = 1 + else: + # Show a limited number of rows. Before modifying query, get the total number of rows that query + # returns so that the total number of pages can be computed. + total_num_rows = query.count() + query = query.limit(self.num_rows_per_page).offset((page_num - 1) * self.num_rows_per_page) + num_pages = int(math.ceil(float(total_num_rows) / self.num_rows_per_page)) + else: + # Defaults. + page_num = 1 + num_pages = 1 + # There are some places in grid templates where it's useful for a grid + # to have its current filter. + self.cur_filter_dict = cur_filter_dict + # Preserve grid state: save current filter and sort key. + if self.preserve_state: + pref_name = text_type(self.__class__.__name__ + self.cur_filter_pref_name) + trans.get_user().preferences[pref_name] = text_type(dumps(cur_filter_dict)) + if sort_key: + pref_name = text_type(self.__class__.__name__ + self.cur_sort_key_pref_name) + trans.get_user().preferences[pref_name] = text_type(dumps(sort_key)) + trans.sa_session.flush() + # Log grid view. + context = text_type(self.__class__.__name__) + params = cur_filter_dict.copy() + params['sort'] = sort_key + params['async'] = ('async' in kwargs) + + # TODO:?? + # commenting this out; when this fn calls session.add( action ) and session.flush the query from this fn + # is effectively 'wiped' out. Nate believes it has something to do with our use of session( autocommit=True ) + # in mapping.py. If you change that to False, the log_action doesn't affect the query + # Below, I'm rendering the template first (that uses query), then calling log_action, then returning the page + # trans.log_action( trans.get_user(), text_type( "grid.view" ), context, params ) + + # Render grid. + def url(*args, **kwargs): + route_name = kwargs.pop('__route_name__', None) + # Only include sort/filter arguments if not linking to another + # page. This is a bit of a hack. + if 'action' in kwargs: + new_kwargs = dict() + else: + new_kwargs = dict(extra_url_args) + # Extend new_kwargs with first argument if found + if len(args) > 0: + new_kwargs.update(args[0]) + new_kwargs.update(kwargs) + # We need to encode item ids + if 'id' in new_kwargs: + id = new_kwargs['id'] + if isinstance(id, list): + new_kwargs['id'] = [trans.security.encode_id(i) for i in id] + else: + new_kwargs['id'] = trans.security.encode_id(id) + # The url_for invocation *must* include a controller and action. + if 'controller' not in new_kwargs: + new_kwargs['controller'] = trans.controller + if 'action' not in new_kwargs: + new_kwargs['action'] = trans.action + if route_name: + return url_for(route_name, **new_kwargs) + return url_for(**new_kwargs) + + self.use_panels = (kwargs.get('use_panels', False) in [True, 'True', 'true']) + self.advanced_search = (kwargs.get('advanced_search', False) in [True, 'True', 'true']) + async_request = ((self.use_async) and (kwargs.get('async', False) in [True, 'True', 'true'])) + # Currently, filling the template returns a str object; this requires decoding the string into a + # unicode object within mako templates. What probably should be done is to return the template as + # utf-8 unicode; however, this would require encoding the object as utf-8 before returning the grid + # results via a controller method, which is require substantial changes. Hence, for now, return grid + # as str. + if not dict_format: + page = trans.fill_template(iff(async_request, self.async_template, self.template), + grid=self, + query=query, + cur_page_num=page_num, + num_pages=num_pages, + num_page_links=self.num_page_links, + default_filter_dict=self.default_filter, + cur_filter_dict=cur_filter_dict, + sort_key=sort_key, + current_item=current_item, + ids=kwargs.get('id', []), + url=url, + status=status, + message=message, + info_text=self.info_text, + use_panels=self.use_panels, + use_hide_message=self.use_hide_message, + advanced_search=self.advanced_search, + show_item_checkboxes=(self.show_item_checkboxes or + kwargs.get('show_item_checkboxes', '') in ['True', 'true']), + # Pass back kwargs so that grid template can set and use args without + # grid explicitly having to pass them. + kwargs=kwargs) + trans.log_action(trans.get_user(), text_type("grid.view"), context, params) + return page + + grid_config = { + 'title' : self.title, + 'url_base' : trans.request.path_url, + 'async' : self.use_async, + 'async_ops' : [], + 'categorical_filters' : {}, + 'filters' : cur_filter_dict, + 'sort_key' : sort_key, + 'show_item_checkboxes' : self.show_item_checkboxes or kwargs.get('show_item_checkboxes', '') in ['True', 'true'], + 'cur_page_num' : page_num, + 'num_pages' : num_pages, + 'num_page_links' : self.num_page_links, + 'status' : status, + 'message' : restore_text(message), + 'global_actions' : [], + 'operations' : [], + 'items' : [], + 'columns' : [], + 'model_class' : str(self.model_class), + 'use_paging' : self.use_paging, + 'legend' : self.legend, + 'current_item_id' : False, + 'use_hide_message' : self.use_hide_message, + 'default_filter_dict' : self.default_filter, + 'advanced_search' : self.advanced_search, + 'info_text' : self.info_text, + 'url' : url(dict()), + 'refresh_frames' : kwargs.get('refresh_frames', []) + } + if current_item: + grid_config['current_item_id'] = current_item.id + for column in self.columns: + href = None + extra = '' + if column.sortable: + if sort_key.endswith(column.key): + if not sort_key.startswith("-"): + href = url(sort=("-" + column.key)) + extra = "↓" + else: + href = url(sort=(column.key)) + extra = "↑" + else: + href = url(sort=column.key) + grid_config['columns'].append({ + 'key' : column.key, + 'visible' : column.visible, + 'nowrap' : column.nowrap, + 'attach_popup' : column.attach_popup, + 'label_id_prefix' : column.label_id_prefix, + 'sortable' : column.sortable, + 'label' : column.label, + 'filterable' : column.filterable, + 'is_text' : isinstance(column, TextColumn), + 'href' : href, + 'extra' : extra + }) + for operation in self.operations: + grid_config['operations'].append({ + 'allow_multiple' : operation.allow_multiple, + 'allow_popup' : operation.allow_popup, + 'target' : operation.target, + 'label' : operation.label, + 'confirm' : operation.confirm, + 'href' : url(**operation.url_args) if isinstance(operation.url_args, dict) else None, + 'global_operation' : False + }) + if operation.allow_multiple: + grid_config['show_item_checkboxes'] = True + if operation.global_operation: + grid_config['global_operation'] = url(** (operation.global_operation())) + for action in self.global_actions: + grid_config['global_actions'].append({ + 'url_args' : url(**action.url_args), + 'label' : action.label, + 'target' : action.target + }) + for operation in [op for op in self.operations if op.async_compatible]: + grid_config['async_ops'].append(operation.label.lower()) + for column in self.columns: + if column.filterable is not None and not isinstance(column, TextColumn): + grid_config['categorical_filters'][column.key] = dict([(filter.label, filter.args) for filter in column.get_accepted_filters()]) + for i, item in enumerate(query): + item_dict = { + 'id' : item.id, + 'encode_id' : trans.security.encode_id(item.id), + 'link' : [], + 'operation_config' : {}, + 'column_config' : {} + } + for column in self.columns: + if column.visible: + link = column.get_link(trans, self, item) + if link: + link = url(**link) + else: + link = None + target = column.target + value = column.get_value(trans, self, item) + if isinstance(value, str): + value = text_type(value, 'utf-8') + value = value.replace('/', '//') + item_dict['column_config'][column.label] = { + 'link' : link, + 'value' : value, + 'target' : target + } + for operation in self.operations: + item_dict['operation_config'][operation.label] = { + 'allowed' : operation.allowed(item), + 'url_args' : url(**operation.get_url_args(item)), + 'target' : operation.target + } + grid_config['items'].append(item_dict) + trans.log_action(trans.get_user(), text_type("grid.view"), context, params) + return grid_config + + def get_ids(self, **kwargs): + id = [] + if 'id' in kwargs: + id = kwargs['id'] + # Coerce ids to list + if not isinstance(id, list): + id = id.split(",") + # Ensure ids are integers + try: + id = list(map(int, id)) + except Exception: + decorators.error("Invalid id") + return id + + # ---- Override these ---------------------------------------------------- + def handle_operation(self, trans, operation, ids, **kwargs): + pass + + def get_current_item(self, trans, **kwargs): + return None + + def build_initial_query(self, trans, **kwargs): + return trans.sa_session.query(self.model_class) + + def apply_query_filter(self, trans, query, **kwargs): + # Applies a database filter that holds for all items in the grid. + # (gvk) Is this method necessary? Why not simply build the entire query, + # including applying filters in the build_initial_query() method? + return query + + +class GridColumn(object): + def __init__(self, label, key=None, model_class=None, method=None, format=None, + link=None, attach_popup=False, visible=True, nowrap=False, + # Valid values for filterable are ['standard', 'advanced', None] + filterable=None, sortable=True, label_id_prefix=None, target=None): + """Create a grid column.""" + self.label = label + self.key = key + self.model_class = model_class + self.method = method + self.format = format + self.link = link + self.target = target + self.nowrap = nowrap + self.attach_popup = attach_popup + self.visible = visible + self.filterable = filterable + # Column must have a key to be sortable. + self.sortable = (self.key is not None and sortable) + self.label_id_prefix = label_id_prefix or '' + + def get_value(self, trans, grid, item): + if self.method: + value = getattr(grid, self.method)(trans, item) + elif self.key: + value = getattr(item, self.key) + else: + value = None + if self.format: + value = self.format(value) + return escape(unicodify(value)) + + def get_link(self, trans, grid, item): + if self.link and self.link(item): + return self.link(item) + return None + + def filter(self, trans, user, query, column_filter): + """ Modify query to reflect the column filter. """ + if column_filter == "All": + pass + if column_filter == "True": + query = query.filter_by(**{self.key: True}) + elif column_filter == "False": + query = query.filter_by(**{self.key: False}) + return query + + def get_accepted_filters(self): + """ Returns a list of accepted filters for this column. """ + accepted_filters_vals = ["False", "True", "All"] + accepted_filters = [] + for val in accepted_filters_vals: + args = {self.key: val} + accepted_filters.append(GridColumnFilter(val, args)) + return accepted_filters + + def sort(self, trans, query, ascending, column_name=None): + """Sort query using this column.""" + if column_name is None: + column_name = self.key + if ascending: + query = query.order_by(self.model_class.table.c.get(column_name).asc()) + else: + query = query.order_by(self.model_class.table.c.get(column_name).desc()) + return query + + +class ReverseSortColumn(GridColumn): + """ Column that reverses sorting; this is useful when the natural sort is descending. """ + + def sort(self, trans, query, ascending, column_name=None): + return GridColumn.sort(self, trans, query, (not ascending), column_name=column_name) + + +class TextColumn(GridColumn): + """ Generic column that employs freetext and, hence, supports freetext, case-independent filtering. """ + + def filter(self, trans, user, query, column_filter): + """ Modify query to filter using free text, case independence. """ + if column_filter == "All": + pass + elif column_filter: + query = query.filter(self.get_filter(trans, user, column_filter)) + return query + + def get_filter(self, trans, user, column_filter): + """ Returns a SQLAlchemy criterion derived from column_filter. """ + if isinstance(column_filter, string_types): + return self.get_single_filter(user, column_filter) + elif isinstance(column_filter, list): + clause_list = [] + for filter in column_filter: + clause_list.append(self.get_single_filter(user, filter)) + return and_(*clause_list) + + def get_single_filter(self, user, a_filter): + """ + Returns a SQLAlchemy criterion derived for a single filter. Single filter + is the most basic filter--usually a string--and cannot be a list. + """ + # Queries that include table joins cannot guarantee that table column names will be + # unique, so check to see if a_filter is of type .. + if self.key.find('.') > -1: + a_key = self.key.split('.')[1] + else: + a_key = self.key + model_class_key_field = getattr(self.model_class, a_key) + return func.lower(model_class_key_field).like("%" + a_filter.lower() + "%") + + def sort(self, trans, query, ascending, column_name=None): + """Sort column using case-insensitive alphabetical sorting.""" + if column_name is None: + column_name = self.key + if ascending: + query = query.order_by(func.lower(self.model_class.table.c.get(column_name)).asc()) + else: + query = query.order_by(func.lower(self.model_class.table.c.get(column_name)).desc()) + return query + + +class DateTimeColumn(TextColumn): + def sort(self, trans, query, ascending, column_name=None): + """Sort query using this column.""" + return GridColumn.sort(self, trans, query, ascending, column_name=column_name) + + +class BooleanColumn(TextColumn): + def sort(self, trans, query, ascending, column_name=None): + """Sort query using this column.""" + return GridColumn.sort(self, trans, query, ascending, column_name=column_name) + + def get_single_filter(self, user, a_filter): + if self.key.find('.') > -1: + a_key = self.key.split('.')[1] + else: + a_key = self.key + model_class_key_field = getattr(self.model_class, a_key) + return model_class_key_field == a_filter + + +class IntegerColumn(TextColumn): + """ + Integer column that employs freetext, but checks that the text is an integer, + so support filtering on integer values. + + IMPORTANT NOTE: grids that use this column type should not include the column + in the cols_to_filter list of MulticolFilterColumn ( i.e., searching on this + column type should not be performed in the grid's standard search - it won't + throw exceptions, but it also will not find what you're looking for ). Grids + that search on this column should use 'filterable="advanced"' so that searching + is only performed in the advanced search component, restricting the search to + the specific column. + + This is useful for searching on object ids or other integer columns. See the + JobIdColumn column in the SpecifiedDateListGrid class in the jobs controller of + the reports webapp for an example. + """ + + def get_single_filter(self, user, a_filter): + model_class_key_field = getattr(self.model_class, self.key) + assert int(a_filter), "The search entry must be an integer" + return model_class_key_field == int(a_filter) + + def sort(self, trans, query, ascending, column_name=None): + """Sort query using this column.""" + return GridColumn.sort(self, trans, query, ascending, column_name=column_name) + + +class CommunityRatingColumn(GridColumn, UsesItemRatings): + """ Column that displays community ratings for an item. """ + + def get_value(self, trans, grid, item): + ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, item, webapp_model=trans.model) + return trans.fill_template("tool_shed_rating.mako", + ave_item_rating=ave_item_rating, + num_ratings=num_ratings, + item_id=trans.security.encode_id(item.id)) + + def sort(self, trans, query, ascending, column_name=None): + def get_foreign_key(source_class, target_class): + """ Returns foreign key in source class that references target class. """ + target_fk = None + for fk in source_class.table.foreign_keys: + if fk.references(target_class.table): + target_fk = fk + break + if not target_fk: + raise RuntimeException("No foreign key found between objects: %s, %s" % source_class.table, target_class.table) + return target_fk + # Get the columns that connect item's table and item's rating association table. + item_rating_assoc_class = getattr(trans.model, '%sRatingAssociation' % self.model_class.__name__) + foreign_key = get_foreign_key(item_rating_assoc_class, self.model_class) + fk_col = foreign_key.parent + referent_col = foreign_key.get_referent(self.model_class.table) + # Do sorting using a subquery. + # Subquery to get average rating for each item. + ave_rating_subquery = trans.sa_session.query(fk_col, + func.avg(item_rating_assoc_class.table.c.rating).label('avg_rating')) \ + .group_by(fk_col).subquery() + # Integrate subquery into main query. + query = query.outerjoin((ave_rating_subquery, referent_col == ave_rating_subquery.columns[fk_col.name])) + # Sort using subquery results; use coalesce to avoid null values. + if not ascending: # TODO: for now, reverse sorting b/c first sort is ascending, and that should be the natural sort. + query = query.order_by(func.coalesce(ave_rating_subquery.c.avg_rating, 0).asc()) + else: + query = query.order_by(func.coalesce(ave_rating_subquery.c.avg_rating, 0).desc()) + return query + + +class OwnerAnnotationColumn(TextColumn, UsesAnnotations): + """ Column that displays and filters item owner's annotations. """ + + def __init__(self, col_name, key, model_class=None, model_annotation_association_class=None, filterable=None): + GridColumn.__init__(self, col_name, key=key, model_class=model_class, filterable=filterable) + self.sortable = False + self.model_annotation_association_class = model_annotation_association_class + + def get_value(self, trans, grid, item): + """ Returns first 150 characters of annotation. """ + annotation = self.get_item_annotation_str(trans.sa_session, item.user, item) + if annotation: + ann_snippet = annotation[:155] + if len(annotation) > 155: + ann_snippet = ann_snippet[:ann_snippet.rfind(' ')] + ann_snippet += "..." + else: + ann_snippet = "" + return escape(ann_snippet) + + def get_single_filter(self, user, a_filter): + """ Filter by annotation and annotation owner. """ + return self.model_class.annotations.any( + and_(func.lower(self.model_annotation_association_class.annotation).like("%" + a_filter.lower() + "%"), + # TODO: not sure why, to filter by owner's annotations, we have to do this rather than + # 'self.model_class.user==self.model_annotation_association_class.user' + self.model_annotation_association_class.table.c.user_id == self.model_class.table.c.user_id)) + + +class CommunityTagsColumn(TextColumn): + """ Column that supports community tags. """ + + def __init__(self, col_name, key, model_class=None, model_tag_association_class=None, filterable=None, grid_name=None): + GridColumn.__init__(self, col_name, key=key, model_class=model_class, nowrap=True, filterable=filterable, sortable=False) + self.model_tag_association_class = model_tag_association_class + # Column-specific attributes. + self.grid_name = grid_name + + def get_value(self, trans, grid, item): + return trans.fill_template("/tagging_common.mako", tag_type="community", trans=trans, user=trans.get_user(), tagged_item=item, elt_context=self.grid_name, + in_form=True, input_size="20", tag_click_fn="add_tag_to_grid_filter", use_toggle_link=True) + + def filter(self, trans, user, query, column_filter): + """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ + if column_filter == "All": + pass + elif column_filter: + query = query.filter(self.get_filter(trans, user, column_filter)) + return query + + def get_filter(self, trans, user, column_filter): + # Parse filter to extract multiple tags. + if isinstance(column_filter, list): + # Collapse list of tags into a single string; this is redundant but effective. TODO: fix this by iterating over tags. + column_filter = ",".join(column_filter) + raw_tags = trans.app.tag_handler.parse_tags(column_filter.encode("utf-8")) + clause_list = [] + for name, value in raw_tags: + if name: + # Filter by all tags. + clause_list.append(self.model_class.tags.any(func.lower(self.model_tag_association_class.user_tname).like("%" + name.lower() + "%"))) + if value: + # Filter by all values. + clause_list.append(self.model_class.tags.any(func.lower(self.model_tag_association_class.user_value).like("%" + value.lower() + "%"))) + return and_(*clause_list) + + +class IndividualTagsColumn(CommunityTagsColumn): + """ Column that supports individual tags. """ + + def get_value(self, trans, grid, item): + return trans.fill_template("/tagging_common.mako", + tag_type="individual", + user=trans.user, + tagged_item=item, + elt_context=self.grid_name, + in_form=True, + input_size="20", + tag_click_fn="add_tag_to_grid_filter", + use_toggle_link=True) + + def get_filter(self, trans, user, column_filter): + # Parse filter to extract multiple tags. + if isinstance(column_filter, list): + # Collapse list of tags into a single string; this is redundant but effective. TODO: fix this by iterating over tags. + column_filter = ",".join(column_filter) + raw_tags = trans.app.tag_handler.parse_tags(column_filter.encode("utf-8")) + clause_list = [] + for name, value in raw_tags: + if name: + # Filter by individual's tag names. + clause_list.append(self.model_class.tags.any(and_(func.lower(self.model_tag_association_class.user_tname).like("%" + name.lower() + "%"), self.model_tag_association_class.user == user))) + if value: + # Filter by individual's tag values. + clause_list.append(self.model_class.tags.any(and_(func.lower(self.model_tag_association_class.user_value).like("%" + value.lower() + "%"), self.model_tag_association_class.user == user))) + return and_(*clause_list) + + +class MulticolFilterColumn(TextColumn): + """ Column that performs multicolumn filtering. """ + + def __init__(self, col_name, cols_to_filter, key, visible, filterable="default"): + GridColumn.__init__(self, col_name, key=key, visible=visible, filterable=filterable) + self.cols_to_filter = cols_to_filter + + def filter(self, trans, user, query, column_filter): + """ Modify query to filter model_class by tag. Multiple filters are ANDed. """ + if column_filter == "All": + return query + if isinstance(column_filter, list): + clause_list = [] + for filter in column_filter: + part_clause_list = [] + for column in self.cols_to_filter: + part_clause_list.append(column.get_filter(trans, user, filter)) + clause_list.append(or_(*part_clause_list)) + complete_filter = and_(*clause_list) + else: + clause_list = [] + for column in self.cols_to_filter: + clause_list.append(column.get_filter(trans, user, column_filter)) + complete_filter = or_(*clause_list) + return query.filter(complete_filter) + + +class OwnerColumn(TextColumn): + """ Column that lists item's owner. """ + + def get_value(self, trans, grid, item): + return item.user.username + + def sort(self, trans, query, ascending, column_name=None): + """ Sort column using case-insensitive alphabetical sorting on item's username. """ + if ascending: + query = query.order_by(func.lower(self.model_class.username).asc()) + else: + query = query.order_by(func.lower(self.model_class.username).desc()) + return query + + +class PublicURLColumn(TextColumn): + """ Column displays item's public URL based on username and slug. """ + + def get_link(self, trans, grid, item): + if item.user.username and item.slug: + return dict(action='display_by_username_and_slug', username=item.user.username, slug=item.slug) + elif not item.user.username: + # TODO: provide link to set username. + return None + elif not item.user.slug: + # TODO: provide link to set slug. + return None + + +class DeletedColumn(GridColumn): + """ Column that tracks and filters for items with deleted attribute. """ + + def get_accepted_filters(self): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = {"active" : "False", "deleted" : "True", "all": "All"} + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = {self.key: val} + accepted_filters.append(GridColumnFilter(label, args)) + return accepted_filters + + def filter(self, trans, user, query, column_filter): + """Modify query to filter self.model_class by state.""" + if column_filter == "All": + pass + elif column_filter in ["True", "False"]: + query = query.filter(self.model_class.deleted == (column_filter == "True")) + return query + + +class StateColumn(GridColumn): + """ + Column that tracks and filters for items with state attribute. + + IMPORTANT NOTE: self.model_class must have a states Bunch or dict if + this column type is used in the grid. + """ + + def get_value(self, trans, grid, item): + return item.state + + def filter(self, trans, user, query, column_filter): + """Modify query to filter self.model_class by state.""" + if column_filter == "All": + pass + elif column_filter in [v for k, v in self.model_class.states.items()]: + query = query.filter(self.model_class.state == column_filter) + return query + + def get_accepted_filters(self): + """Returns a list of accepted filters for this column.""" + all = GridColumnFilter('all', {self.key : 'All'}) + accepted_filters = [all] + for k, v in self.model_class.states.items(): + args = {self.key: v} + accepted_filters.append(GridColumnFilter(v, args)) + return accepted_filters + + +class SharingStatusColumn(GridColumn): + """ Grid column to indicate sharing status. """ + + def get_value(self, trans, grid, item): + # Delete items cannot be shared. + if item.deleted: + return "" + # Build a list of sharing for this item. + sharing_statuses = [] + if item.users_shared_with: + sharing_statuses.append("Shared") + if item.importable: + sharing_statuses.append("Accessible") + if item.published: + sharing_statuses.append("Published") + return ", ".join(sharing_statuses) + + def get_link(self, trans, grid, item): + if not item.deleted and (item.users_shared_with or item.importable or item.published): + return dict(operation="share or publish", id=item.id) + return None + + def filter(self, trans, user, query, column_filter): + """ Modify query to filter histories by sharing status. """ + if column_filter == "All": + pass + elif column_filter: + if column_filter == "private": + query = query.filter(self.model_class.users_shared_with == null()) + query = query.filter(self.model_class.importable == false()) + elif column_filter == "shared": + query = query.filter(self.model_class.users_shared_with != null()) + elif column_filter == "accessible": + query = query.filter(self.model_class.importable == true()) + elif column_filter == "published": + query = query.filter(self.model_class.published == true()) + return query + + def get_accepted_filters(self): + """ Returns a list of accepted filters for this column. """ + accepted_filter_labels_and_vals = odict() + accepted_filter_labels_and_vals["private"] = "private" + accepted_filter_labels_and_vals["shared"] = "shared" + accepted_filter_labels_and_vals["accessible"] = "accessible" + accepted_filter_labels_and_vals["published"] = "published" + accepted_filter_labels_and_vals["all"] = "All" + accepted_filters = [] + for label, val in accepted_filter_labels_and_vals.items(): + args = {self.key: val} + accepted_filters.append(GridColumnFilter(label, args)) + return accepted_filters + + +class GridOperation(object): + def __init__(self, label, key=None, condition=None, allow_multiple=True, allow_popup=True, + target=None, url_args=None, async_compatible=False, confirm=None, + global_operation=None): + self.label = label + self.key = key + self.allow_multiple = allow_multiple + self.allow_popup = allow_popup + self.condition = condition + self.target = target + self.url_args = url_args + self.async_compatible = async_compatible + # if 'confirm' is set, then ask before completing the operation + self.confirm = confirm + # specify a general operation that acts on the full grid + # this should be a function returning a dictionary with parameters + # to pass to the URL, similar to GridColumn links: + # global_operation=(lambda: dict(operation="download") + self.global_operation = global_operation + + def get_url_args(self, item): + if self.url_args: + if hasattr(self.url_args, '__call__'): + url_args = self.url_args(item) + else: + url_args = dict(self.url_args) + url_args['id'] = item.id + return url_args + else: + return dict(operation=self.label, id=item.id) + + def allowed(self, item): + if self.condition: + return bool(self.condition(item)) + else: + return True + + +class DisplayByUsernameAndSlugGridOperation(GridOperation): + """ Operation to display an item by username and slug. """ + + def get_url_args(self, item): + return {'action' : 'display_by_username_and_slug', 'username' : item.user.username, 'slug' : item.slug} + + +class GridAction(object): + def __init__(self, label=None, url_args=None, target=None): + self.label = label + self.url_args = url_args + self.target = target + + +class GridColumnFilter(object): + def __init__(self, label, args=None): + self.label = label + self.args = args + + def get_url_args(self): + rval = {} + for k, v in self.args.items(): + rval["f-" + k] = v + return rval From 7961dcc75ca8f9932b0c1f674347dbaf91190ce6 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:20:42 -0500 Subject: [PATCH 04/56] Strip mako support from galaxy grid helper --- lib/galaxy/web/framework/helpers/grids.py | 30 ----------------------- 1 file changed, 30 deletions(-) diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index fbfc11faf38..a78fd3d8dff 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -24,8 +24,6 @@ class Grid(object): exposed = True model_class = None show_item_checkboxes = False - template = "grid_base.mako" - async_template = "grid_base_async.mako" use_async = False use_hide_message = True global_actions = [] @@ -65,7 +63,6 @@ class Grid(object): # FIXME: pretty sure this is only here to pass along, can likely be eliminated status = kwargs.get('status', None) message = kwargs.get('message', None) - dict_format = kwargs.get('dict_format', False) # Build a base filter and sort key that is the combination of the saved state and defaults. # Saved state takes preference over defaults. base_filter = {} @@ -286,33 +283,6 @@ class Grid(object): # utf-8 unicode; however, this would require encoding the object as utf-8 before returning the grid # results via a controller method, which is require substantial changes. Hence, for now, return grid # as str. - if not dict_format: - page = trans.fill_template(iff(async_request, self.async_template, self.template), - grid=self, - query=query, - cur_page_num=page_num, - num_pages=num_pages, - num_page_links=self.num_page_links, - default_filter_dict=self.default_filter, - cur_filter_dict=cur_filter_dict, - sort_key=sort_key, - current_item=current_item, - ids=kwargs.get('id', []), - url=url, - status=status, - message=message, - info_text=self.info_text, - use_panels=self.use_panels, - use_hide_message=self.use_hide_message, - advanced_search=self.advanced_search, - show_item_checkboxes=(self.show_item_checkboxes or - kwargs.get('show_item_checkboxes', '') in ['True', 'true']), - # Pass back kwargs so that grid template can set and use args without - # grid explicitly having to pass them. - kwargs=kwargs) - trans.log_action(trans.get_user(), text_type("grid.view"), context, params) - return page - grid_config = { 'title' : self.title, 'url_base' : trans.request.path_url, From f600a1dd99998e9415215a045bafd3a47ffb4975 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:23:07 -0500 Subject: [PATCH 05/56] Remove comma from root controller config dict --- lib/galaxy/webapps/galaxy/controllers/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index dcf876cfb9f..497e8b4ccb8 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -57,7 +57,7 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): 'toolbox' : app.toolbox.to_dict(trans, in_panel=False), 'toolbox_in_panel' : app.toolbox.to_dict(trans), 'message_box_visible' : app.config.message_box_visible, - 'show_inactivity_warning' : app.config.user_activation_on and trans.user and not trans.user.active, + 'show_inactivity_warning' : app.config.user_activation_on and trans.user and not trans.user.active } # TODO: move to user From 24cd0893e5f9ad5aab1befa2aa311ba9b597302a Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:24:01 -0500 Subject: [PATCH 06/56] Link jobs mako to legacy reports module --- lib/galaxy/webapps/reports/controllers/jobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index 73804c3fa16..fa46025b5a6 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -17,8 +17,8 @@ from sqlalchemy import and_, not_, or_ from galaxy import model, util from galaxy.web.base.controller import BaseUIController, web -from galaxy.web.framework.helpers import grids from galaxy.webapps.reports.controllers.query import ReportQueryBuilder +from galaxy.webapps.reports.framework import grids log = logging.getLogger(__name__) @@ -179,7 +179,6 @@ class SpecifiedDateListGrid(grids.Grid): use_async = False model_class = model.Job title = "Jobs" - template = '/webapps/reports/grid.mako' default_sort_key = "id" columns = [ JobIdColumn("Id", From a8f59df445cbe36f9cdc281f2fe5aa91a809be5c Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:24:59 -0500 Subject: [PATCH 07/56] Same for worflows grid in reports --- lib/galaxy/webapps/reports/controllers/workflows.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/workflows.py b/lib/galaxy/webapps/reports/controllers/workflows.py index fe6e06dfac0..74d362f1f86 100644 --- a/lib/galaxy/webapps/reports/controllers/workflows.py +++ b/lib/galaxy/webapps/reports/controllers/workflows.py @@ -18,12 +18,12 @@ from sqlalchemy import and_ from galaxy import model, util from galaxy.web.base.controller import BaseUIController, web -from galaxy.web.framework.helpers import grids from galaxy.webapps.reports.controllers.jobs import ( get_spark_time, sorter ) from galaxy.webapps.reports.controllers.query import ReportQueryBuilder +from galaxy.webapps.reports.framework import grids log = logging.getLogger(__name__) @@ -81,7 +81,6 @@ class SpecifiedDateListGrid(grids.Grid): use_async = False model_class = model.StoredWorkflow title = "Workflows" - template = '/webapps/reports/grid.mako' default_sort_key = "name" columns = [ WorkflowNameColumn("Name", From 47578b641fadee5b9dd2d0e65fe06da1dfb5ccfd Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:26:56 -0500 Subject: [PATCH 08/56] Redirect for admin toolshed grids --- lib/galaxy/webapps/tool_shed/controllers/repository.py | 2 +- lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 7b5606b90bb..1e27e8dbbab 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -21,7 +21,7 @@ from galaxy import ( from galaxy.tools.repositories import ValidationContext from galaxy.web.base.controller import BaseUIController from galaxy.web.form_builder import CheckboxField, SelectField -from galaxy.web.framework.helpers import grids +from galaxy.webapps.reports.framework import grids from galaxy.webapps.tool_shed.util import ratings_util from tool_shed.capsule import capsule_manager from tool_shed.dependencies.repository import relation_builder diff --git a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py index b20b6f89ca9..3f312794a9c 100644 --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -3,7 +3,8 @@ import logging from galaxy import util from galaxy.model import tool_shed_install from galaxy.web import url_for -from galaxy.web.framework.helpers import grids, iff +from galaxy.web.framework.helpers import iff +from galaxy.webapps.reports.framework import grids from tool_shed.util import repository_util log = logging.getLogger(__name__) From b726cda3a7abbc49a0532b942fcf6923475d51df Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:28:27 -0500 Subject: [PATCH 09/56] More redirects for toolshed grids --- lib/tool_shed/grids/admin_grids.py | 3 ++- lib/tool_shed/grids/repository_review_grids.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tool_shed/grids/admin_grids.py b/lib/tool_shed/grids/admin_grids.py index 5f5310a2c77..64d45388466 100644 --- a/lib/tool_shed/grids/admin_grids.py +++ b/lib/tool_shed/grids/admin_grids.py @@ -3,7 +3,8 @@ import logging from markupsafe import escape from sqlalchemy import and_ -from galaxy.web.framework.helpers import grids, time_ago +from galaxy.web.framework.helpers import time_ago +from galaxy.webapps.reports.framework import grids from galaxy.webapps.tool_shed import model from tool_shed.grids.repository_grids import CategoryGrid, RepositoryGrid from tool_shed.util import hg_util diff --git a/lib/tool_shed/grids/repository_review_grids.py b/lib/tool_shed/grids/repository_review_grids.py index 95b6881d557..abad59f5d02 100644 --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -3,7 +3,7 @@ import logging from markupsafe import escape from sqlalchemy import and_, false, null, or_, true -from galaxy.web.framework.helpers import grids +from galaxy.webapps.reports.framework import grids from galaxy.webapps.tool_shed import model from tool_shed.grids.repository_grids import RepositoryGrid from tool_shed.util import hg_util, metadata_util From d4dda34f1214df3389d779e9b835d7557a7499c8 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:28:54 -0500 Subject: [PATCH 10/56] Remove reports default grid mako --- templates/webapps/reports/grid.mako | 1 - 1 file changed, 1 deletion(-) delete mode 100644 templates/webapps/reports/grid.mako diff --git a/templates/webapps/reports/grid.mako b/templates/webapps/reports/grid.mako deleted file mode 100644 index 6a41b688c5e..00000000000 --- a/templates/webapps/reports/grid.mako +++ /dev/null @@ -1 +0,0 @@ -<%inherit file="/grid_base.mako"/> From d930b114558814945bd0d5f054904318c2f83b35 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:29:57 -0500 Subject: [PATCH 11/56] Move grid makos from galaxy to reports app --- templates/{ => webapps/reports}/grid_base.mako | 0 .../{grid_base_async.mako => webapps/reports/grid_base_asyn.mako} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename templates/{ => webapps/reports}/grid_base.mako (100%) rename templates/{grid_base_async.mako => webapps/reports/grid_base_asyn.mako} (100%) diff --git a/templates/grid_base.mako b/templates/webapps/reports/grid_base.mako similarity index 100% rename from templates/grid_base.mako rename to templates/webapps/reports/grid_base.mako diff --git a/templates/grid_base_async.mako b/templates/webapps/reports/grid_base_asyn.mako similarity index 100% rename from templates/grid_base_async.mako rename to templates/webapps/reports/grid_base_asyn.mako From 6c974dd81b0f6fb2dec6c65d1a0c3a98983cda5b Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:32:53 -0500 Subject: [PATCH 12/56] Redirect makos to new grid helper location in reports --- templates/webapps/reports/grid_base.mako | 2 +- .../reports/{grid_base_asyn.mako => grid_base_async.mako} | 0 templates/webapps/tool_shed/category/grid.mako | 2 +- templates/webapps/tool_shed/category/valid_grid.mako | 2 +- templates/webapps/tool_shed/common/grid_common.mako | 2 +- templates/webapps/tool_shed/repository_review/grid.mako | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename templates/webapps/reports/{grid_base_asyn.mako => grid_base_async.mako} (100%) diff --git a/templates/webapps/reports/grid_base.mako b/templates/webapps/reports/grid_base.mako index 4b9a7a84a92..e70f4327f36 100644 --- a/templates/webapps/reports/grid_base.mako +++ b/templates/webapps/reports/grid_base.mako @@ -1,5 +1,5 @@ <%! - from galaxy.web.framework.helpers.grids import TextColumn + from galaxy.webapps.reports.framework.grids import TextColumn def inherit(context): kwargs = context.get( 'kwargs', {} ) diff --git a/templates/webapps/reports/grid_base_asyn.mako b/templates/webapps/reports/grid_base_async.mako similarity index 100% rename from templates/webapps/reports/grid_base_asyn.mako rename to templates/webapps/reports/grid_base_async.mako diff --git a/templates/webapps/tool_shed/category/grid.mako b/templates/webapps/tool_shed/category/grid.mako index e56a03bca8f..9c2ab498d7c 100644 --- a/templates/webapps/tool_shed/category/grid.mako +++ b/templates/webapps/tool_shed/category/grid.mako @@ -1,5 +1,5 @@ <%inherit file="/base.mako"/> -<%namespace name="grid_base" file="/grid_base.mako" import="*" /> +<%namespace name="grid_base" file="/webapps/reports/grid_base.mako" import="*" /> <%namespace name="grid_common" file="../common/grid_common.mako" import="*" /> <%def name="insert()"> diff --git a/templates/webapps/tool_shed/category/valid_grid.mako b/templates/webapps/tool_shed/category/valid_grid.mako index 2872e12b39d..962e2018f36 100644 --- a/templates/webapps/tool_shed/category/valid_grid.mako +++ b/templates/webapps/tool_shed/category/valid_grid.mako @@ -1,5 +1,5 @@ <%inherit file="/base.mako"/> -<%namespace name="grid_base" file="/grid_base.mako" import="*" /> +<%namespace name="grid_base" file="/webapps/reports/grid_base.mako" import="*" /> <%namespace name="grid_common" file="../common/grid_common.mako" import="*" /> <%def name="insert()"> diff --git a/templates/webapps/tool_shed/common/grid_common.mako b/templates/webapps/tool_shed/common/grid_common.mako index 627e19a6064..af724b5d90d 100644 --- a/templates/webapps/tool_shed/common/grid_common.mako +++ b/templates/webapps/tool_shed/common/grid_common.mako @@ -1,5 +1,5 @@ <%! - from galaxy.web.framework.helpers.grids import TextColumn, StateColumn, GridColumnFilter + from galaxy.webapps.reports.framework.grids import TextColumn, StateColumn, GridColumnFilter from galaxy.web.framework.helpers import iff %> diff --git a/templates/webapps/tool_shed/repository_review/grid.mako b/templates/webapps/tool_shed/repository_review/grid.mako index 6a41b688c5e..9b13abc893a 100644 --- a/templates/webapps/tool_shed/repository_review/grid.mako +++ b/templates/webapps/tool_shed/repository_review/grid.mako @@ -1 +1 @@ -<%inherit file="/grid_base.mako"/> +<%inherit file="/webapps/reports/framework/grid_base.mako"/> From d2752dc7b7977f6d887fe7b7c88f53141a2b46bf Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 9 Dec 2017 16:36:46 -0500 Subject: [PATCH 13/56] Add init file to framework module --- lib/galaxy/webapps/reports/framework/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/galaxy/webapps/reports/framework/__init__.py diff --git a/lib/galaxy/webapps/reports/framework/__init__.py b/lib/galaxy/webapps/reports/framework/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 96b9b151aea1236cc754bdabc9c1ebe29f23533c Mon Sep 17 00:00:00 2001 From: guerler Date: Sun, 10 Dec 2017 10:18:58 -0500 Subject: [PATCH 14/56] Remove dict format from legacy grid helper --- lib/galaxy/webapps/reports/framework/grids.py | 167 +++--------------- lib/tool_shed/grids/repository_grids.py | 2 +- 2 files changed, 25 insertions(+), 144 deletions(-) diff --git a/lib/galaxy/webapps/reports/framework/grids.py b/lib/galaxy/webapps/reports/framework/grids.py index fbfc11faf38..6d875bb9a03 100644 --- a/lib/galaxy/webapps/reports/framework/grids.py +++ b/lib/galaxy/webapps/reports/framework/grids.py @@ -65,7 +65,6 @@ class Grid(object): # FIXME: pretty sure this is only here to pass along, can likely be eliminated status = kwargs.get('status', None) message = kwargs.get('message', None) - dict_format = kwargs.get('dict_format', False) # Build a base filter and sort key that is the combination of the saved state and defaults. # Saved state takes preference over defaults. base_filter = {} @@ -286,149 +285,31 @@ class Grid(object): # utf-8 unicode; however, this would require encoding the object as utf-8 before returning the grid # results via a controller method, which is require substantial changes. Hence, for now, return grid # as str. - if not dict_format: - page = trans.fill_template(iff(async_request, self.async_template, self.template), - grid=self, - query=query, - cur_page_num=page_num, - num_pages=num_pages, - num_page_links=self.num_page_links, - default_filter_dict=self.default_filter, - cur_filter_dict=cur_filter_dict, - sort_key=sort_key, - current_item=current_item, - ids=kwargs.get('id', []), - url=url, - status=status, - message=message, - info_text=self.info_text, - use_panels=self.use_panels, - use_hide_message=self.use_hide_message, - advanced_search=self.advanced_search, - show_item_checkboxes=(self.show_item_checkboxes or - kwargs.get('show_item_checkboxes', '') in ['True', 'true']), - # Pass back kwargs so that grid template can set and use args without - # grid explicitly having to pass them. - kwargs=kwargs) - trans.log_action(trans.get_user(), text_type("grid.view"), context, params) - return page - - grid_config = { - 'title' : self.title, - 'url_base' : trans.request.path_url, - 'async' : self.use_async, - 'async_ops' : [], - 'categorical_filters' : {}, - 'filters' : cur_filter_dict, - 'sort_key' : sort_key, - 'show_item_checkboxes' : self.show_item_checkboxes or kwargs.get('show_item_checkboxes', '') in ['True', 'true'], - 'cur_page_num' : page_num, - 'num_pages' : num_pages, - 'num_page_links' : self.num_page_links, - 'status' : status, - 'message' : restore_text(message), - 'global_actions' : [], - 'operations' : [], - 'items' : [], - 'columns' : [], - 'model_class' : str(self.model_class), - 'use_paging' : self.use_paging, - 'legend' : self.legend, - 'current_item_id' : False, - 'use_hide_message' : self.use_hide_message, - 'default_filter_dict' : self.default_filter, - 'advanced_search' : self.advanced_search, - 'info_text' : self.info_text, - 'url' : url(dict()), - 'refresh_frames' : kwargs.get('refresh_frames', []) - } - if current_item: - grid_config['current_item_id'] = current_item.id - for column in self.columns: - href = None - extra = '' - if column.sortable: - if sort_key.endswith(column.key): - if not sort_key.startswith("-"): - href = url(sort=("-" + column.key)) - extra = "↓" - else: - href = url(sort=(column.key)) - extra = "↑" - else: - href = url(sort=column.key) - grid_config['columns'].append({ - 'key' : column.key, - 'visible' : column.visible, - 'nowrap' : column.nowrap, - 'attach_popup' : column.attach_popup, - 'label_id_prefix' : column.label_id_prefix, - 'sortable' : column.sortable, - 'label' : column.label, - 'filterable' : column.filterable, - 'is_text' : isinstance(column, TextColumn), - 'href' : href, - 'extra' : extra - }) - for operation in self.operations: - grid_config['operations'].append({ - 'allow_multiple' : operation.allow_multiple, - 'allow_popup' : operation.allow_popup, - 'target' : operation.target, - 'label' : operation.label, - 'confirm' : operation.confirm, - 'href' : url(**operation.url_args) if isinstance(operation.url_args, dict) else None, - 'global_operation' : False - }) - if operation.allow_multiple: - grid_config['show_item_checkboxes'] = True - if operation.global_operation: - grid_config['global_operation'] = url(** (operation.global_operation())) - for action in self.global_actions: - grid_config['global_actions'].append({ - 'url_args' : url(**action.url_args), - 'label' : action.label, - 'target' : action.target - }) - for operation in [op for op in self.operations if op.async_compatible]: - grid_config['async_ops'].append(operation.label.lower()) - for column in self.columns: - if column.filterable is not None and not isinstance(column, TextColumn): - grid_config['categorical_filters'][column.key] = dict([(filter.label, filter.args) for filter in column.get_accepted_filters()]) - for i, item in enumerate(query): - item_dict = { - 'id' : item.id, - 'encode_id' : trans.security.encode_id(item.id), - 'link' : [], - 'operation_config' : {}, - 'column_config' : {} - } - for column in self.columns: - if column.visible: - link = column.get_link(trans, self, item) - if link: - link = url(**link) - else: - link = None - target = column.target - value = column.get_value(trans, self, item) - if isinstance(value, str): - value = text_type(value, 'utf-8') - value = value.replace('/', '//') - item_dict['column_config'][column.label] = { - 'link' : link, - 'value' : value, - 'target' : target - } - for operation in self.operations: - item_dict['operation_config'][operation.label] = { - 'allowed' : operation.allowed(item), - 'url_args' : url(**operation.get_url_args(item)), - 'target' : operation.target - } - grid_config['items'].append(item_dict) + page = trans.fill_template(iff(async_request, self.async_template, self.template), + grid=self, + query=query, + cur_page_num=page_num, + num_pages=num_pages, + num_page_links=self.num_page_links, + default_filter_dict=self.default_filter, + cur_filter_dict=cur_filter_dict, + sort_key=sort_key, + current_item=current_item, + ids=kwargs.get('id', []), + url=url, + status=status, + message=message, + info_text=self.info_text, + use_panels=self.use_panels, + use_hide_message=self.use_hide_message, + advanced_search=self.advanced_search, + show_item_checkboxes=(self.show_item_checkboxes or + kwargs.get('show_item_checkboxes', '') in ['True', 'true']), + # Pass back kwargs so that grid template can set and use args without + # grid explicitly having to pass them. + kwargs=kwargs) trans.log_action(trans.get_user(), text_type("grid.view"), context, params) - return grid_config + return page def get_ids(self, **kwargs): id = [] diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index d4228adb59e..f51b01a8cee 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -7,7 +7,7 @@ from sqlalchemy import and_, false, or_, true import tool_shed.grids.util as grids_util import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc -from galaxy.web.framework.helpers import grids +from galaxy.webapps.reports.framework import grids from galaxy.webapps.tool_shed import model from tool_shed.util import hg_util, metadata_util, repository_util From 1c8ed722c221de5baaf437085c0b3e84b1ee5689 Mon Sep 17 00:00:00 2001 From: guerler Date: Sun, 10 Dec 2017 10:29:47 -0500 Subject: [PATCH 15/56] Copy legacy grid code to reports app, redirect imports --- client/galaxy/scripts/{reports_webapp => reports}/run_stats.js | 0 static/scripts/reports_webapp/run_stats.js | 1 - templates/webapps/reports/grid_base.mako | 2 +- templates/webapps/reports/run_stats.mako | 2 +- 4 files changed, 2 insertions(+), 3 deletions(-) rename client/galaxy/scripts/{reports_webapp => reports}/run_stats.js (100%) delete mode 100644 static/scripts/reports_webapp/run_stats.js diff --git a/client/galaxy/scripts/reports_webapp/run_stats.js b/client/galaxy/scripts/reports/run_stats.js similarity index 100% rename from client/galaxy/scripts/reports_webapp/run_stats.js rename to client/galaxy/scripts/reports/run_stats.js diff --git a/static/scripts/reports_webapp/run_stats.js b/static/scripts/reports_webapp/run_stats.js deleted file mode 100644 index 951aa619c1a..00000000000 --- a/static/scripts/reports_webapp/run_stats.js +++ /dev/null @@ -1 +0,0 @@ -define("reports_webapp/run_stats",["exports","../libs/d3"],function(t,e){"use strict";function r(t,e){return new Date(t.getFullYear(),t.getMonth(),t.getDate()-e,t.getHours(),t.getMinutes(),t.getSeconds(),t.getMilliseconds())}function a(t,e){return new Date(t.getFullYear(),t.getMonth(),t.getDate(),t.getHours()-e,t.getMinutes(),t.getSeconds(),t.getMilliseconds())}function n(){var t=new Date;return new Date(t.getUTCFullYear(),t.getUTCMonth(),t.getUTCDate(),t.getUTCHours(),0,0)}Object.defineProperty(t,"__esModule",{value:!0}),t.create_chart=function(t,i,l,s){var c,p=t,g=[],u=n();for(c=0;c<24;c++)g.push(a(u,c));var d=[];for(c=0;c<30;c++)d.push(r(u,c));var f={top:60,right:30,bottom:50,left:60},h=0;"hours"==l?h=12.5:"days"==l&&(h=10);var x,v=300+f.left+f.right,m=1.75;x=0!==o.max(p)?150/o.max(p):1;var b=150+f.top+f.bottom;$(".charts").css("height",b*m);var y=o.select("#"+i).attr("width",v).attr("height",b).attr("preserveAspectRatio","xMidYMin").attr("viewBox","0 0 "+v+" "+b).on("click",function(){var t=o.select(this).attr("class");t=t.split(" "),o.selectAll("."+t[0]).filter("."+t[1]).style("cursor","zoom-in").transition().duration(750).attr("height",b).attr("width",v),o.select(this).style("cursor","default").transition().duration(750).attr("height",b*m).attr("width",v*m)}),_=y.selectAll("g").data(p).enter().append("g").attr("transform",function(t,e){var r=f.left;return"translate("+(r+=+e*h)+","+f.top+")"}).on("mouseenter",function(t){for(var r=1,a=t;a>=10;)a/=10,r++;var n=4*r+10;o.select(e.event.target.parentElement).select(".tool_tip").select("text").attr("transform","translate( "+(f.left-5)+", "+(150-t*x+f.top+10)+" )").attr("visibility","visible").text(t),o.select(e.event.target.parentElement).select(".tool_tip").attr("width",n+"px").attr("height","15px").select("rect").attr("transform","translate( "+(f.left-n)+", "+(150-t*x+f.top)+" )").attr("width",n+"px").attr("height","15px").attr("fill","#ebd9b2")}).on("mouseleave",function(t){o.select(e.event.target.parentElement).select(".tool_tip").select("text").attr("visibility","hidden"),o.select(e.event.target.parentElement).select(".tool_tip").select("rect").attr("width","0").attr("height","0").attr("fill","").text(t)});y.append("g").append("text").attr("class","title").attr("text-anchor","end").attr("transform",function(){return"translate( 300,15 )"}).text(s),y.append("g").attr("class","axis").append("path").attr("class","x").attr("d",function(){var t=f.left,e=f.top+150;return"M"+t+" "+e+" L "+(t+300)+" "+e});var w=o.scale.linear().range([150,0]),M=o.svg.axis().scale(w).orient("left").tickFormat(function(t){return o.round(t*o.max(p),0)});y.append("g").attr("class","y axis").attr("id","y_"+i).attr("text-anchor","end").attr("transform","translate( "+f.left+","+f.top+")").call(M).select(".domain"),y.append("g").append("text").attr("class","ax_title").attr("transform",function(){var t=o.select("#y_"+i).node();return"translate("+(f.left-t.getBoundingClientRect().width-5)+","+(f.top+t.getBoundingClientRect().height/2-30)+")rotate(-90)"}).text("Number of Jobs"),_.append("rect").attr("y",function(t){return 150-t*x}).attr("height",function(t){return t*x}).attr("width",h-1);var k=!1;if("hours"==l){_.append("line").attr("x1",0).attr("y1",0).attr("x2",0).attr("y2",3).attr("stroke","black").attr("stroke-width",1).attr("pointer-events","none").attr("transform",function(){return"translate( "+h/2+", 150)"}),_.append("text").attr("fill","rgb(0,0,0)").attr("transform","translate( 10, 160 )").text(function(t,e){return g[e].getHours()<10?"0"+String(g[e].getHours()):g[e].getHours()});var D="";k=!1,_.append("line").attr("x1",0).attr("y1",0).attr("x2",0).attr("y2",function(t,e){var r=0;return g[e].getDate()!=D&&(k?r=20:(r=27,k=!0),D=g[e].getDate()),r}).attr("stroke","black").attr("stroke-width",1).attr("pointer-events","none").attr("transform",function(){return"translate( 0, 150)"}),D="";var B="";k=!1,_.append("text").attr("fill","rgb(0,0,0)").attr("pointer-events","none").text(function(t,e){var r="";return g[e].getDate()!=B&&(r=String(g[e].toLocaleString("en-us",{month:"long"})),r+=" "+String(g[e].getDate()),B=g[e].getDate()),r}).attr("transform",function(t,e){var r=150,a=o.select(this).node().getBBox().width;return g[e].getDate()!=D&&(k?r+=18:(r+=26,k=!0),D=g[e].getDate()),"translate( "+(a+2)+", "+r+" )"})}else if("days"==l){_.append("line").attr("x1",0).attr("y1",0).attr("x2",0).attr("y2",3).attr("stroke","black").attr("stroke-width",1).attr("pointer-events","none").attr("transform",function(){return"translate( "+h/2+", 150)"}),_.append("text").attr("fill","rgb(0,0,0)").attr("transform","translate( 9, 160 )").text(function(t,e){return d[e].getDate()<10?"0"+String(d[e].getDate()):d[e].getDate()});var C="";k=!1,_.append("line").attr("x1",0).attr("y1",0).attr("x2",0).attr("y2",function(t,e){var r=0;return d[e].getMonth()!=C&&(k?r=20:(r=27,k=!0),C=d[e].getMonth()),r}).attr("stroke","black").attr("stroke-width",1).attr("pointer-events","none").attr("transform",function(){return"translate( 0, 150)"}),C="";var S="";k=!1,_.append("text").attr("fill","rgb(0,100,0)").attr("pointer-events","none").text(function(t,e){var r="";return d[e].getMonth()!=S&&(r=String(d[e].toLocaleString("en-us",{month:"long"})),r+=" "+String(d[e].getFullYear()),S=d[e].getMonth()),r}).attr("transform",function(t,e){var r=150,a=o.select(this).node().getBBox().width;return d[e].getMonth()!=C&&(k?r+=18:(r+=26,k=!0),C=d[e].getMonth()),"translate( "+(a+2)+", "+r+" )"})}y.append("g").attr("class","tool_tip").append("rect"),y.select(".tool_tip").append("text"),"jc_dy_chart"!=i&&"jc_hr_chart"!=i||o.select("#"+i).attr("height",b*m).attr("width",v*m).style("cursor","default")},t.create_histogram=function(t,r,a){var n=t,i=1.75,l={top:60,right:30,bottom:50,left:60},s=150+l.top+l.bottom,c=300+l.left+l.right,p=o.scale.linear().domain([0,o.max(n)]).range([0,300]);n=o.layout.histogram().bins(p.ticks(20))(n);for(var g=[],u=0;u=1;)a/=10,r++;var n=4*r+10;o.select(e.event.target.parentElement).select(".tool_tip").select("text").attr("transform","translate( "+(l.left-5)+", "+(150-t.length*d+l.top+10)+" )").attr("visibility","visible").text(t.length),o.select(e.event.target.parentElement).select(".tool_tip").attr("width",n+"px").attr("height","15px").select("rect").attr("transform","translate( "+(l.left-n)+", "+(150-t.length*d+l.top)+" )").attr("width",n+"px").attr("height","15px").attr("fill","#ebd9b2")}).on("mouseleave",function(){o.select(e.event.target.parentElement).select(".tool_tip").select("text").attr("visibility","hidden"),o.select(e.event.target.parentElement).select(".tool_tip").select("rect").attr("width","0").attr("height","0").attr("fill","")});x=void 0===n[0]?1:p(n[0].dx),v.append("rect").attr("x",1).attr("width",x-1).attr("height",function(t){return 150-f(t.y)});var m=o.svg.axis().scale(p).orient("bottom").tickFormat(function(t){var e=Math.floor(t/60),r=Math.floor(t-60*e);return e<10&&(e="0"+e),r<10&&(r="0"+r),e+":"+r});h.append("g").attr("class","x axis").attr("id","x_"+r).attr("transform","translate( "+l.left+","+(150+l.top)+")").call(m),h.append("g").append("text").attr("class","ax_title").attr("transform",function(){var t=o.select("#x_"+r).node();return"translate("+(l.left+t.getBoundingClientRect().width/2+30)+","+(l.top+150+t.getBoundingClientRect().height+10)+")"}).text("ETA - hrs:mins");var b=o.svg.axis().scale(f).orient("left");h.append("g").attr("class","y axis").attr("id","y_"+r).attr("transform","translate( "+l.left+","+l.top+")").call(b),h.append("g").append("text").attr("class","ax_title").attr("transform",function(){var t=o.select("#y_"+r).node();return"translate("+(l.left-t.getBoundingClientRect().width-5)+","+(l.top+t.getBoundingClientRect().height/2-30)+")rotate(-90)"}).text("Number of Jobs"),h.append("g").attr("class","tool_tip").append("rect"),h.select(".tool_tip").append("text")};var o=function(t){if(t&&t.__esModule)return t;var e={};if(null!=t)for(var r in t)Object.prototype.hasOwnProperty.call(t,r)&&(e[r]=t[r]);return e.default=t,e}(e)}); \ No newline at end of file diff --git a/templates/webapps/reports/grid_base.mako b/templates/webapps/reports/grid_base.mako index e70f4327f36..82f92321c30 100644 --- a/templates/webapps/reports/grid_base.mako +++ b/templates/webapps/reports/grid_base.mako @@ -64,7 +64,7 @@ ## load javascript