From e08dae34628d31f6685f9f7e031e2085c024f63b Mon Sep 17 00:00:00 2001 From: balto Date: Mon, 9 May 2016 16:43:18 +0200 Subject: [PATCH 01/11] Creation of the history category that contains two pages: history and dataset per type/user. --- .../webapps/reports/controllers/history.py | 199 ++++++++++++++++++ .../reports/history_and_dataset_per_user.mako | 74 +++++++ .../reports/history_and_dataset_type.mako | 73 +++++++ templates/webapps/reports/index.mako | 11 + 4 files changed, 357 insertions(+) create mode 100644 lib/galaxy/webapps/reports/controllers/history.py create mode 100644 templates/webapps/reports/history_and_dataset_per_user.mako create mode 100644 templates/webapps/reports/history_and_dataset_type.mako diff --git a/lib/galaxy/webapps/reports/controllers/history.py b/lib/galaxy/webapps/reports/controllers/history.py new file mode 100644 index 00000000000..121c8e959c2 --- /dev/null +++ b/lib/galaxy/webapps/reports/controllers/history.py @@ -0,0 +1,199 @@ + +import collections +import calendar, operator +import sqlalchemy as sa +import logging +from datetime import datetime, date, timedelta +from time import strftime + +import galaxy.model +import pkg_resources +from galaxy import util +from galaxy.model.orm import and_, not_, or_ +from galaxy.web.base.controller import BaseUIController, web +from markupsafe import escape + +pkg_resources.require( "SQLAlchemy >= 0.4" ) +log = logging.getLogger( __name__ ) + + +def int_to_octet (size): + try: + size = float (size) + except ValueError: + return "???" + except TypeError: + if size is None: + return "0 o" + return "???" + units = ("o", "Ko", "Mo", "Go", "To") + no_unit = 0 + while (size >= 1000): + size /= 1000. + no_unit += 1 + try: + return "%.2f %s"%( size, units[no_unit] ) + except IndexError: + return "%.0f %s"%( size * ( ( no_unit-len( units )+1 ) * 1000. ), units[-1] ) + +class History( BaseUIController ): + """ + Class defining functions used by reports to make requests to get + informations and fill templates before beeing displayed. + The name of function must be the same as as the field "action" of + the "href" dict, in data.mako (templates/webapps/reports). + """ + + + @web.expose + def history_and_dataset_per_user( self, trans, **kwd ): + """ + fill history_and_dataset_per_user.mako template with: + - user email + - the number of history and their size + - the number of dataset + """ + + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) + user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) + + # sort by history space, or by user mail or by number of history/dataset + sort_by = kwd.get( 'sorting', 'User' ) + sorting = 0 if sort_by == 'User' else 1 if sort_by == "HSort" else 2 if sort_by == "DSort" else 3 + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + + # select count (h.id) as history, u.email as email + # from history h, galaxy_user u + # where h.user_id = u.id and h.deleted='f' + # group by email order by email desc + histories = sa.select( ( sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ), + galaxy.model.User.table.c.email.label( 'email' ) ), + from_obj=[ sa.outerjoin( galaxy.model.History.table, galaxy.model.User.table ) ], + whereclause=and_( galaxy.model.History.table.c.user_id==galaxy.model.User.table.c.id, + galaxy.model.History.table.c.deleted=='f' ), + group_by=[ 'email' ], + order_by=[ sa.desc( 'email' ), 'history' ] ) + + # select u.email, count(d.id) + # from galaxy_user u, dataset d, history_dataset_association hd,history h + # where d.id=hd.dataset_id and h.id=hd.history_id and u.id = h.user_id and h.deleted='f' + # group by u.email; + datasets = sa.select( ( sa.func.count( galaxy.model.Dataset.table.c.id ).label( 'dataset' ), + sa.func.sum( galaxy.model.Dataset.table.c.total_size ).label( 'size' ), + galaxy.model.User.table.c.email.label( 'email' ) ), + from_obj=[ galaxy.model.User.table, + galaxy.model.Dataset.table, + galaxy.model.HistoryDatasetAssociation.table, + galaxy.model.History.table], + whereclause=and_(galaxy.model.Dataset.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,\ + galaxy.model.History.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.history_id, \ + galaxy.model.History.table.c.user_id==galaxy.model.User.table.c.id, + galaxy.model.History.table.c.deleted=='f'), + group_by=[ 'email' ] ) + + # execute requests, replace None fields by "Unknown" + # transform lists to dict with email as key and + # number of (history/dataset)/size of history as value + histories = dict( [ ( _.email if _.email is not None else "Unknown", \ + int( _.history ) ) for _ in histories.execute() ] ) + datasets = dict( [ ( _.email if _.email is not None else "Unknown", \ + (int( _.dataset ), int( _.size )) ) for _ in datasets.execute () ] ) + + sorting_functions = [ + lambda first, second: descending if first[0].lower () > second[0].lower () else -descending, \ + lambda first, second: descending if histories.get(first, 0) < histories.get(second, 0) else -descending, \ + lambda first, second: descending if datasets.get(first, [0])[0] < datasets.get(second, [0])[0] else -descending, \ + lambda first, second: descending if datasets.get(first, [0,0])[1] < datasets.get(second, [0,0])[1] else -descending + ] + + # fetch all users + users = list(set(histories.keys()) | set(datasets.keys())) + + # sort users depending on sort function, defined by user choices + users.sort(sorting_functions[sorting]) + if user_cutoff > 0: + users = users[:user_cutoff] + + # to keep ordered + data = collections.OrderedDict() + for user in users: + dataset = datasets.get(user, [0, 0]) + history = histories.get(user, 0) + data[user] = ("%d (%s)"%( history, int_to_octet( dataset[1] ) ), dataset[0] ) + + return trans.fill_template( '/webapps/reports/history_and_dataset_per_user.mako', + data=data, + user_cutoff=user_cutoff, + sorting=sorting, + descending=descending, + message=message ) + + @web.expose + def history_and_dataset_type( self, trans, **kwd ): + """ + fill history_and_dataset_type.mako template with: + - the name of history + - the number of dataset foreach type + """ + + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) + user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + user_selection = kwd.get( 'user_selection', None) + + # select d.state, h.name + # from dataset d, history h , history_dataset_association hda + # where hda.history_id=h.id and hda.dataset_id=d.id order by h.state; + from_obj = [ galaxy.model.Dataset.table, galaxy.model.History.table, galaxy.model.HistoryDatasetAssociation.table] + if user_selection is not None: + from_obj.append( galaxy.model.User.table ) + whereclause = and_( galaxy.model.Dataset.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,\ + galaxy.model.History.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.history_id, \ + galaxy.model.User.table.c.id==galaxy.model.History.table.c.user_id, \ + galaxy.model.User.table.c.email==user_selection ) + else : + whereclause = and_(galaxy.model.Dataset.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,\ + galaxy.model.History.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.history_id) + histories = sa.select( ( galaxy.model.Dataset.table.c.state.label( 'state' ), + galaxy.model.History.table.c.name.label( 'name' ) ), + from_obj=from_obj, + whereclause=whereclause, + order_by = [ 'name' ] ) + + # execute requests, replace None fields by "Unknown" + data = [ ( _.name if _.name is not None else "NoNamedHistory", \ + _.state ) for _ in histories.execute() ] + + # sort by names descending or ascending + data.sort(lambda first, second: descending if first[0].lower () > second[0].lower () else -descending) + + # fetch names in the first list and status in the second + if data: + names, status = zip( *tuple(data) ) + else: + names, status = [], [] + + + possible_status = {"ok" :0, "upload": 1, "paused": 2, "queued": 3, "error": 4, "discarded": 5} + number_of_possible_status = len( possible_status ) + 1 # + 1 to handle unknown status! + + # to keep ordered + datas = collections.OrderedDict() + for no, name in enumerate (names): + if name not in datas: + if user_cutoff > 0: + if len( datas ) == user_cutoff: + break + # creation of a list containing the number of each status + datas[name] = ['-'] * number_of_possible_status + # to not execute it several times, we put it in a variable... + no_status = possible_status.get(status[no], 6) + if datas[name][no_status] == '-': + datas[name][no_status] = 0 + datas[name][no_status] += 1 + + return trans.fill_template( '/webapps/reports/history_and_dataset_type.mako', + data=datas, + user_cutoff=user_cutoff, + descending=descending, + message=message ) \ No newline at end of file diff --git a/templates/webapps/reports/history_and_dataset_per_user.mako b/templates/webapps/reports/history_and_dataset_per_user.mako new file mode 100644 index 00000000000..dfe83016502 --- /dev/null +++ b/templates/webapps/reports/history_and_dataset_per_user.mako @@ -0,0 +1,74 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, 'done' )} +%endif + +
+

Per-user number of History and Datasets

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by + %if sorting == 0: + Users + %elif sorting == 1: + number of History + %elif sorting == 2: + number of Dataset + %else: + History Space + %endif +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + + +
+ +

+
+
+ + %if data: + + + + + + <% odd = False%> + %for user in data: + %if odd: + + %else: + + %endif + + + + + <% odd = not odd %> + %endfor + %endif +
UserNumber of HistoryNumber of Dataset
${user}${data[user][0]}${data[user][1]}
+
diff --git a/templates/webapps/reports/history_and_dataset_type.mako b/templates/webapps/reports/history_and_dataset_type.mako new file mode 100644 index 00000000000..2d80246b604 --- /dev/null +++ b/templates/webapps/reports/history_and_dataset_type.mako @@ -0,0 +1,73 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, 'done' )} +%endif + +
+

History and Datasets types

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by History name +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + +
+ +

+
+
+ + %if data: + + + + + + + + + + + + + <% odd = False%> + %for name in data: + %if odd: + + %else: + + %endif + + + + + + + + + + + + <% odd = not odd %> + %endfor + %endif +
History nameDataset in state 'ok'Dataset in state 'upload'Dataset pausedDataset queuedDataset in errorDataset discarded
${name}${data[name][0]}${data[name][1]}${data[name][2]}${data[name][3]}${data[name][4]}${data[name][5]}
+
diff --git a/templates/webapps/reports/index.mako b/templates/webapps/reports/index.mako index ff179034839..b3e56e50881 100644 --- a/templates/webapps/reports/index.mako +++ b/templates/webapps/reports/index.mako @@ -73,6 +73,17 @@
+
+ History +
+
+
+
History and dataset per user
+
History and dataset type
+
+
+
+
Workflows
From cd2b61a3a1426cdd62f9af3fe506affa3c097688 Mon Sep 17 00:00:00 2001 From: balto Date: Mon, 9 May 2016 16:49:00 +0200 Subject: [PATCH 02/11] Creation of the tools controller and each methods for each template. --- .../webapps/reports/controllers/tools.py | 371 ++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 lib/galaxy/webapps/reports/controllers/tools.py diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py new file mode 100644 index 00000000000..d985749ade6 --- /dev/null +++ b/lib/galaxy/webapps/reports/controllers/tools.py @@ -0,0 +1,371 @@ + +import collections +import calendar, operator +import sqlalchemy as sa +import logging +from datetime import datetime, date, timedelta +from time import strftime +import collections + +import galaxy.model +import pkg_resources +from galaxy import util +from galaxy.model.orm import and_, not_, or_ +from galaxy.web.base.controller import BaseUIController, web +from markupsafe import escape + +pkg_resources.require( "SQLAlchemy >= 0.4" ) +log = logging.getLogger( __name__ ) + + +def int_to_octet (size): + try: + size = float (size) + except ValueError: + return "???" + except TypeError: + if size is None: + return "0 o" + return "???" + units = ("o", "Ko", "Mo", "Go", "To") + no_unit = 0 + while (size >= 1000): + size /= 1000. + no_unit += 1 + try: + return "%.2f %s"%( size, units[no_unit] ) + except IndexError: + return "%.0f %s"%( size * ( ( no_unit-len( units )+1 ) * 1000. ), units[-1] ) + +class Tools( BaseUIController ): + """ + Class defining functions used by reports to make requests to get + informations and fill templates before beeing displayed. + The name of function must be the same as as the field "action" of + the "href" dict, in data.mako (templates/webapps/reports). + """ + + def formated (self, date, colored = False): + splited = str (date).split(',') + if len (splited) == 2: + returned = "%s %dH" % (splited[0], int (splited[1].split (':')[0])) + if colored: + return '' + returned + '' + return returned + else: + splited = tuple ([float (_) for _ in str (date).split (':')]) + if splited[0]: + returned = '%d h. %d min.' % splited[:2] + if colored: + return '' + returned + '' + return returned + if splited[1]: + return "%d min. %d sec." % splited[1:3] + return "%.1f sec." % splited[2] + + @web.expose + def tools_and_job_state( self, trans, **kwd ): + """ + fill tools_and_job_state_peer_month.mako template with + - the name of the tool + - the number of jobs using this tool in state 'ok' + - the number of jobs using this tool in error + """ + + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) + user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) + + # sort by history space, or by user mail or by number of history/dataset + sort_by = kwd.get( 'sorting', 'Tool' ) + sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2 + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + sort_functions = (lambda first, second: descending if first.lower () > second.lower () else -descending, \ + lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > \ + tools_and_jobs_ok.get( second ) else descending, \ + lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > \ + tools_and_jobs_error.get( second, 0 ) else descending) + + + data = collections.OrderedDict() + + # select count(id), tool_id from job where state='ok' group by tool_id; + tools_and_jobs_ok = sa.select( (galaxy.model.Job.table.c.tool_id .label( 'tool' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=(galaxy.model.Job.table.c.state=='ok'), + group_by=[ 'tool' ] ) + + # select count(id), tool_id from job where state='error' group by tool_id; + tools_and_jobs_error = sa.select( (galaxy.model.Job.table.c.tool_id .label( 'tool' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=(galaxy.model.Job.table.c.state=='error'), + group_by=[ 'tool' ] ) + + tools_and_jobs_ok = dict( list( tools_and_jobs_ok.execute () ) ) + tools_and_jobs_error = dict( list( tools_and_jobs_error.execute () ) ) + + # select each job name one time + tools = list (set(tools_and_jobs_ok.keys ()) | set (tools_and_jobs_error.keys ())) + tools.sort (sort_functions[ sorting ]) + + for tool in tools: + data[tool] = (str( tools_and_jobs_ok.get (tool, '-') ), str( tools_and_jobs_error.get (tool, '-') ) ) + + return trans.fill_template( '/webapps/reports/tools_and_job_state.mako', + data=data, + user_cutoff=user_cutoff, + sorting=sorting, + descending=descending, + message=message ) + + @web.expose + def tools_and_job_state_peer_month (self, trans, **kwd ): + """ + fill tools_and_job_state_peer_month.mako template with + - the name of the tool + - the number of jobs using this tool in state 'ok' + - the number of jobs using this tool in error + """ + + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) + user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) + + # sort by history space, or by user mail or by number of history/dataset + sort_by = kwd.get( 'sorting', 'Tool' ) + sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2 + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + tool = kwd.get( 'tool', None) + + if tool is None: + raise TypeError ("Tool can't be None") + + sort_functions = (lambda first, second: descending if first.lower () > second.lower () else -descending, \ + lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > \ + tools_and_jobs_ok.get( second ) else descending, \ + lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > \ + tools_and_jobs_error.get( second, 0 ) else descending) + + data = collections.OrderedDict() + + # select count(id), create_time from job where state='ok' and tool_id=$tool group by date; + date_and_jobs_ok = sa.select( (sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=and_( galaxy.model.Job.table.c.state=='ok', + galaxy.model.Job.table.c.tool_id==tool ), + group_by=[ 'date' ] ) + + # select count(id), create_time from job where state='error' and tool_id=$tool group by date; + date_and_jobs_error = sa.select( (sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=and_( galaxy.model.Job.table.c.state=='error', + galaxy.model.Job.table.c.tool_id==tool ), + group_by=[ 'date' ] ) + + date_and_jobs_ok = dict( list( date_and_jobs_ok.execute () ) ) + date_and_jobs_error = dict( list( date_and_jobs_error.execute () ) ) + + # select each date + dates = list (set(date_and_jobs_ok.keys ()) | set (date_and_jobs_error.keys ())) + dates.sort (reverse=True) + for date in dates: + date_key = date.strftime( "%B %Y" ) + if date_key not in data: + data[date_key] = [int( date_and_jobs_ok.get (date, 0) ), int( date_and_jobs_error.get (date, 0) ) ] + else : + data[date_key][0] += int( date_and_jobs_ok.get ( date, 0 ) ) + data[date_key][1] += int( date_and_jobs_error.get( date, 0 ) ) + + + return trans.fill_template( '/webapps/reports/tools_and_job_state_peer_month.mako', + data=data, + tool=tool, + user_cutoff=user_cutoff, + message=message ) + + @web.expose + def tool_execution_time (self, trans, **kwd): + """ + Fill th template tool_execution_time.mako with informations: + - Tool name + - Tool average execution time + - last job execution time + - min and max execution time + """ + #~ liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois (?) + user_cutoff = int (kwd.get ("user_cutoff", 60)) + sort_by = kwd.get ("sort_by", "tool") + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + sort_by = 0 if sort_by == "tool" else 1 if sort_by == "avg" else 2 if sort_by == "min" else 3 + color = True if kwd.get ("color", '') == "True" else False + + data = {} + ordered_data = collections.OrderedDict() + field_sort = lambda first, second, field: descending if data[first][field] < data[second][field] else -descending + sort_functions = [ + lambda first, second: -descending if first.lower () < second.lower () else descending, + lambda first, second: field_sort (first, second, "avg"), + lambda first, second: field_sort (first, second, "min"), + lambda first, second: field_sort (first, second, "max")] + + jobs_times = sa.select ( (galaxy.model.Job.table.c.tool_id.label( "name" ), + galaxy.model.Job.table.c.create_time.label ("create_time"), + galaxy.model.Job.table.c.update_time.label ("update_time"), + galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + from_obj = [galaxy.model.Job.table]) + + jobs_times = [(name, (create, update, time)) for name, create, update, time in jobs_times.execute()] + for tool, attr in jobs_times: + if tool not in data: + data[tool] = {"last": [(attr[1], attr[0])], + "avg": [attr[2]]} + else: + data[tool]["last"].append ((attr[1], attr[0])) + data[tool]["avg"].append (attr[2]) + + for tool in data: + data[tool]["min"] = min (data[tool]["avg"]) + data[tool]["max"] = max (data[tool]["avg"]) + last = max (data[tool]["last"]) + data[tool]["last"] = last[0] - last[1] + data[tool]["avg"] = sum (data[tool]["avg"], timedelta()) / len (data[tool]["avg"]) + + + tools = data.keys () + if user_cutoff: + tools = tools[:user_cutoff] + tools.sort (sort_functions[sort_by]) + for tool in tools: + ordered_data[tool] = {"min": self.formated (data[tool]["min"], color), + "max": self.formated (data[tool]["max"], color), + "avg": self.formated (data[tool]["avg"], color), + "last": self.formated (data[tool]["last"], color)} + + return trans.fill_template( '/webapps/reports/tool_execution_time.mako', + data=ordered_data, + descending=descending, + user_cutoff=user_cutoff, + sort_by=sort_by ) + + @web.expose + def tool_execution_time_peer_month (self, trans, **kwd): + """ + Fill th template tool_execution_time_peer_month.mako with informations: + - Tool average execution time + - last job execution time + - min and max execution time + """ + #~ liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois (?) + user_cutoff = int (kwd.get ("user_cutoff", 60)) + sort_by = kwd.get ("sort_by", "month") + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + sort_by = 0 if sort_by == "month" else 1 if sort_by == "min" else 2 if sort_by == "max" else 3 + tool = kwd.get ("tool", None) + color = True if kwd.get ("color", '') == "True" else False + + if tool is None: + raise ValueError ("Tool can't be None") + + ordered_data = collections.OrderedDict() + sort_functions = [(lambda first, second, i=i: descending if first[i] < second[i] else -descending) for i in range(4)] + + jobs_times = sa.select ((sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time ).label ('date'), + sa.func.max (galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + sa.func.avg (galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + sa.func.min (galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time)), + from_obj = [galaxy.model.Job.table], + whereclause = galaxy.model.Job.table.c.tool_id == tool, + group_by = ['date']) + + months = list (jobs_times.execute()) + months.sort (sort_functions[sort_by]) + if user_cutoff: + months = months[:user_cutoff] + + for month in months: + ordered_data[str(month[0]).split(' ')[0][:-3]] = (self.formated (month[1], color), + self.formated (month[2], color), + self.formated (month[3], color)) + + return trans.fill_template( '/webapps/reports/tool_execution_time_peer_month.mako', + data=ordered_data, + tool=tool, + descending=descending, + user_cutoff=user_cutoff, + sort_by=sort_by ) + + @web.expose + def tool_error_messages (self, trans, **kwd): + tool_name = kwd.get ("tool", None) + descending = 1 if kwd.get ("descending", 'desc') == "desc" else -1 + sort_by = 0 if kwd.get ("sort_by", "time") == "time" else 1 + cutoff = int (kwd.get ("user_cutoff", 60)) + sort_functions = (lambda _a, _b: -descending if counter[_a][1] > counter[_b][1] else descending, + lambda _a, _b: -descending if counter[_a][0] > counter[_b][0] else descending) + + if tool_name is None: + raise ValueError ("Tool can't be none") + tool_errors = [[unicode (a), b] for a, b in \ + sa.select ((galaxy.model.Job.table.c.stderr, galaxy.model.Job.table.c.create_time), + from_obj = [galaxy.model.Job.table], + whereclause=and_(galaxy.model.Job.table.c.tool_id==tool_name, + galaxy.model.Job.table.c.state=='error')).execute ()] + + counter = {} + for error in tool_errors: + try: + error[0] = unicode(error[0].decode ("utf-8")) + # encoding tested: + # latin-1 ; iso-8859-1 ; alien ; cenc ; cp037 ; cp437 ; base64 ; utf-8 ; utf-16 ; ascii ; hex + except UnicodeEncodeError: + for no, lettre in enumerate (error[0]): + try: + str (lettre.decode ("utf-8")) + except UnicodeEncodeError: + try: + error[0] = error[0].replace (error[0][no], '?') + except UnicodeEncodeError: + error[0] = "This error contains special character and can't be displayed." + break + if error[0] in counter: + counter[error[0]][0] += 1 + else: + counter[error[0]] = [1, error[1]] + + data = collections.OrderedDict () + keys = counter.keys () + if cutoff: + keys = keys[:cutoff] + keys.sort (sort_functions [sort_by]) + + spaces = [' ', '\t', ' '] + for key in keys: + new_key = '
'.join ([_ for _ in key.split ('\n') if _ and _ not in spaces]) + if len (new_key) >= 100: + to_replace = [] + words = key.split ('\n') + for word in words: + if word in to_replace: + continue + if words.count (word) > 1: + print word + to_replace.append (word) + for word in to_replace: + sentence = ("
"+word) * 2 + count = 2 + while sentence+"
"+word in new_key: + sentence += "
"+word + count += 1 + print sentence, count + if sentence in new_key: + new_key = new_key.replace (sentence, '
' + word + " [this line in %d times]"%(count)) + data[new_key] = counter[key] + + return trans.fill_template ("/webapps/reports/tool_error_messages.mako", + data = data, + descending = descending, + tool_name=tool_name, + sort_by=sort_by, + user_cutoff=cutoff) \ No newline at end of file From e4d8c7d88556a201a91ac21b67d08feecd57f819 Mon Sep 17 00:00:00 2001 From: balto Date: Tue, 10 May 2016 08:45:06 +0200 Subject: [PATCH 03/11] Implementation of tools-related pages. --- templates/webapps/reports/index.mako | 17 ++++ .../webapps/reports/tool_error_messages.mako | 87 +++++++++++++++++++ .../webapps/reports/tool_execution_time.mako | 77 ++++++++++++++++ .../tool_execution_time_peer_month.mako | 75 ++++++++++++++++ .../webapps/reports/tools_and_job_state.mako | 82 +++++++++++++++++ .../tools_and_job_state_peer_month.mako | 44 ++++++++++ 6 files changed, 382 insertions(+) create mode 100644 templates/webapps/reports/tool_error_messages.mako create mode 100644 templates/webapps/reports/tool_execution_time.mako create mode 100644 templates/webapps/reports/tool_execution_time_peer_month.mako create mode 100644 templates/webapps/reports/tools_and_job_state.mako create mode 100644 templates/webapps/reports/tools_and_job_state_peer_month.mako diff --git a/templates/webapps/reports/index.mako b/templates/webapps/reports/index.mako index b3e56e50881..8d2f6dea8ef 100644 --- a/templates/webapps/reports/index.mako +++ b/templates/webapps/reports/index.mako @@ -82,6 +82,23 @@
History and dataset type
+ + + +
+
+
+ Tools +
+
+
+
History and dataset per user
+
History and dataset type
+
+
+ + +
diff --git a/templates/webapps/reports/tool_error_messages.mako b/templates/webapps/reports/tool_error_messages.mako new file mode 100644 index 00000000000..1d94e271f74 --- /dev/null +++ b/templates/webapps/reports/tool_error_messages.mako @@ -0,0 +1,87 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, 'done' )} +%endif + +
+

${tool_name} in error's stderr message

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by number of error of each type +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + + by: + +
+ +

+
+
+ + %if data: + + + + + + <% odd = False%> + %for error in data: + %if odd: + + %else: + + %endif + %if len (error.split ('
')) < 7: + + %else: + + + %endif + + + <% odd = not odd %> + %endfor + %endif +
Tool errorNumber of error like thisDate of this error
${error} + + ${data[error][0]}${str(data[error][1])[:11]}
+
diff --git a/templates/webapps/reports/tool_execution_time.mako b/templates/webapps/reports/tool_execution_time.mako new file mode 100644 index 00000000000..a2d797a2336 --- /dev/null +++ b/templates/webapps/reports/tool_execution_time.mako @@ -0,0 +1,77 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + + +
+

Time execution peer tool

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by + %if sort_by == 0: + Tool + %elif sort_by == 1: + average time + %elif sort_by == 2: + min time + %elif sort_by == 3: + max time + %endif +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + + + + Highlight big times of execution
+ +

+
+
+ + %if data: + + + + + + + + <% odd = False%> + %for tool in data: + %if odd: + + %else: + + %endif + + + + + + <% odd = not odd %> + %endfor + %endif +
ToolAverage time of executionLast job's time of executionMin time of executionMax time of execution
+ ${tool} + ${data[tool]["avg"]}${data[tool]["last"]}${data[tool]["min"]}${data[tool]["max"]}
+
diff --git a/templates/webapps/reports/tool_execution_time_peer_month.mako b/templates/webapps/reports/tool_execution_time_peer_month.mako new file mode 100644 index 00000000000..5ade9ec8c04 --- /dev/null +++ b/templates/webapps/reports/tool_execution_time_peer_month.mako @@ -0,0 +1,75 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + + +
+

Time execution for ${tool} peer month

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by + %if sort_by == 0: + Month + %elif sort_by == 1: + min time + %elif sort_by == 2: + max time + %elif sort_by == 3: + average time + %endif +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + + + + Highlight big times of execution
+ +

+
+
+ + %if data: + + + + + + + <% odd = False%> + %for month in data: + %if odd: + + %else: + + %endif + + + + <% odd = not odd %> + %endfor + %endif +
ToolMin time of executionMax time of executionAverage time of execution
+ ${month} + ${data[month][2]}${data[month][0]} + ${data[month][1]}
+
diff --git a/templates/webapps/reports/tools_and_job_state.mako b/templates/webapps/reports/tools_and_job_state.mako new file mode 100644 index 00000000000..c2fcddc509b --- /dev/null +++ b/templates/webapps/reports/tools_and_job_state.mako @@ -0,0 +1,82 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, 'done' )} +%endif + +
+

Per-tool number of Jobs in function of the state

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by + %if sorting == 0: + Tool + %elif sorting == 1: + state Ok + %elif sorting == 2: + state error + %endif +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + + +
+ +

+
+
+ + %if data: + + + + + + <% odd = False%> + %for tool in data: + %if odd: + + %else: + + %endif + + + + <% odd = not odd %> + %endfor + %endif +
ToolJobs okJob in error
+ %if data[tool][0] + data[tool][1] != "--": + ${tool} + %else: + ${tool} + %endif + ${data[tool][0]} + %if data[tool][1] != '-': + ${data[tool][1]} + %else: + - + %endif +
+
diff --git a/templates/webapps/reports/tools_and_job_state_peer_month.mako b/templates/webapps/reports/tools_and_job_state_peer_month.mako new file mode 100644 index 00000000000..924272d5350 --- /dev/null +++ b/templates/webapps/reports/tools_and_job_state_peer_month.mako @@ -0,0 +1,44 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, 'done' )} +%endif + +
+

Number of Jobs Ok or in error peer month for ${tool}

+

Listed in descending by month

+ + %if data: + + + + + + <% odd = False%> + %for month in data: + %if odd: + + %else: + + %endif + + + + <% odd = not odd %> + %endfor + %endif +
MonthOkError
${month} + %if data[month][0] == 0: + - + %else: + ${data[month][0]} + %endif + + %if data[month][1] == 0: + - + %else: + ${data[month][1]} + %endif +
+
From fdd778f244b6f647aa8c56e133c3226bbed8ed72 Mon Sep 17 00:00:00 2001 From: balto Date: Tue, 10 May 2016 08:54:08 +0200 Subject: [PATCH 04/11] Implementation of a new page listing histories per users. --- .../webapps/reports/controllers/users.py | 30 +++++++++ .../webapps/reports/history_per_user.mako | 66 +++++++++++++++++++ templates/webapps/reports/index.mako | 7 +- 3 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 templates/webapps/reports/history_per_user.mako diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index d373f992290..c28f221ea01 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -184,3 +184,33 @@ class Users( BaseUIController, ReportQueryBuilder ): users=users, user_cutoff=user_cutoff, message=message ) + + @web.expose + def history_per_user( self, trans, **kwd ): + + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) + user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) + sorting = 0 if kwd.get( 'sorting', 'User' ) == 'User' else 1 + descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + sorting_functions = [ + lambda first, second: descending if first[0].lower () > second[0].lower () else -descending, \ + lambda first, second: descending if first[1] < second[1] else -descending] + + req = sa.select( ( sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ), + galaxy.model.User.table.c.username.label( 'username' ) ), + from_obj=[ sa.outerjoin( galaxy.model.History.table, galaxy.model.User.table ) ], + whereclause=galaxy.model.History.table.c.user_id==galaxy.model.User.table.c.id, + group_by=[ 'username' ], + order_by=[ sa.desc( 'username' ), 'history' ] ) + + histories = [ (_.username if _.username is not None else "Unknown", _.history) for _ in req.execute() ] + histories.sort( sorting_functions[ sorting ] ) + if user_cutoff != 0: + histories = histories[:user_cutoff] + + return trans.fill_template( '/webapps/reports/history_per_user.mako', + histories=histories, + user_cutoff=user_cutoff, + sorting=sorting, + descending=descending, + message=message ) diff --git a/templates/webapps/reports/history_per_user.mako b/templates/webapps/reports/history_per_user.mako new file mode 100644 index 00000000000..d3703437c4e --- /dev/null +++ b/templates/webapps/reports/history_per_user.mako @@ -0,0 +1,66 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> + +%if message: + ${render_msg( message, 'done' )} +%endif + +
+

Per-user number of History

+

Listed in + %if descending == 1: + descending + %else: + ascending + %endif + order by + %if sorting == 0: + Users + %else: + Number + %endif +

+ + + + +
+
+

+ Top shown (0 = all). +
+ Sort: + + +
+ +

+
+
+ + %if histories: + + + + + <% ctr = 0 %> + %for history in histories: + %if ctr % 2 == 1: + + %else: + + %endif + + + + <% ctr += 1 %> + %endfor + %endif +
UserNumber of History
${history[0]}${history[1]}
+
diff --git a/templates/webapps/reports/index.mako b/templates/webapps/reports/index.mako index 8d2f6dea8ef..4ae602fde99 100644 --- a/templates/webapps/reports/index.mako +++ b/templates/webapps/reports/index.mako @@ -82,9 +82,6 @@
History and dataset type
- - -
@@ -96,9 +93,6 @@
History and dataset type
- - -
@@ -121,6 +115,7 @@
Registered users
Date of last login
User disk usage
+
Number of History per user
From 71fc940c8f048d3c3b4e929320bd895c2361e342 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 10 May 2016 09:38:03 -0400 Subject: [PATCH 05/11] Gets reports/controllers/history.py up to passing style. --- .../webapps/reports/controllers/history.py | 124 +++++++++--------- 1 file changed, 60 insertions(+), 64 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/history.py b/lib/galaxy/webapps/reports/controllers/history.py index 121c8e959c2..f0757027bd6 100644 --- a/lib/galaxy/webapps/reports/controllers/history.py +++ b/lib/galaxy/webapps/reports/controllers/history.py @@ -1,15 +1,11 @@ - import collections -import calendar, operator import sqlalchemy as sa import logging -from datetime import datetime, date, timedelta -from time import strftime import galaxy.model import pkg_resources from galaxy import util -from galaxy.model.orm import and_, not_, or_ +from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseUIController, web from markupsafe import escape @@ -17,9 +13,9 @@ pkg_resources.require( "SQLAlchemy >= 0.4" ) log = logging.getLogger( __name__ ) -def int_to_octet (size): +def int_to_octet(size): try: - size = float (size) + size = float(size) except ValueError: return "???" except TypeError: @@ -32,9 +28,10 @@ def int_to_octet (size): size /= 1000. no_unit += 1 try: - return "%.2f %s"%( size, units[no_unit] ) + return "%.2f %s" % (size, units[no_unit]) except IndexError: - return "%.0f %s"%( size * ( ( no_unit-len( units )+1 ) * 1000. ), units[-1] ) + return "%.0f %s" % (size * ((no_unit - len(units) + 1) * 1000.), units[-1] ) + class History( BaseUIController ): """ @@ -44,7 +41,6 @@ class History( BaseUIController ): the "href" dict, in data.mako (templates/webapps/reports). """ - @web.expose def history_and_dataset_per_user( self, trans, **kwd ): """ @@ -53,10 +49,9 @@ class History( BaseUIController ): - the number of history and their size - the number of dataset """ - message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) - + # sort by history space, or by user mail or by number of history/dataset sort_by = kwd.get( 'sorting', 'User' ) sorting = 0 if sort_by == 'User' else 1 if sort_by == "HSort" else 2 if sort_by == "DSort" else 3 @@ -66,47 +61,49 @@ class History( BaseUIController ): # from history h, galaxy_user u # where h.user_id = u.id and h.deleted='f' # group by email order by email desc - histories = sa.select( ( sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ), - galaxy.model.User.table.c.email.label( 'email' ) ), - from_obj=[ sa.outerjoin( galaxy.model.History.table, galaxy.model.User.table ) ], - whereclause=and_( galaxy.model.History.table.c.user_id==galaxy.model.User.table.c.id, - galaxy.model.History.table.c.deleted=='f' ), - group_by=[ 'email' ], + histories = sa.select( + (sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ), + galaxy.model.User.table.c.email.label( 'email' )), + from_obj=[sa.outerjoin(galaxy.model.History.table, galaxy.model.User.table)], + whereclause=and_(galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, + galaxy.model.History.table.c.deleted == 'f'), + group_by=['email'], order_by=[ sa.desc( 'email' ), 'history' ] ) - # select u.email, count(d.id) - # from galaxy_user u, dataset d, history_dataset_association hd,history h - # where d.id=hd.dataset_id and h.id=hd.history_id and u.id = h.user_id and h.deleted='f' - # group by u.email; - datasets = sa.select( ( sa.func.count( galaxy.model.Dataset.table.c.id ).label( 'dataset' ), - sa.func.sum( galaxy.model.Dataset.table.c.total_size ).label( 'size' ), - galaxy.model.User.table.c.email.label( 'email' ) ), - from_obj=[ galaxy.model.User.table, - galaxy.model.Dataset.table, - galaxy.model.HistoryDatasetAssociation.table, - galaxy.model.History.table], - whereclause=and_(galaxy.model.Dataset.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,\ - galaxy.model.History.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.history_id, \ - galaxy.model.History.table.c.user_id==galaxy.model.User.table.c.id, - galaxy.model.History.table.c.deleted=='f'), + # select u.email, count(d.id) + # from galaxy_user u, dataset d, history_dataset_association hd,history h + # where d.id=hd.dataset_id and h.id=hd.history_id and u.id = h.user_id and h.deleted='f' + # group by u.email; + datasets = sa.select( + (sa.func.count( galaxy.model.Dataset.table.c.id ).label( 'dataset' ), + sa.func.sum( galaxy.model.Dataset.table.c.total_size ).label( 'size' ), + galaxy.model.User.table.c.email.label( 'email' ) ), + from_obj=[ galaxy.model.User.table, + galaxy.model.Dataset.table, + galaxy.model.HistoryDatasetAssociation.table, + galaxy.model.History.table], + whereclause=and_(galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id, + galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id, + galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, + galaxy.model.History.table.c.deleted == 'f'), group_by=[ 'email' ] ) # execute requests, replace None fields by "Unknown" # transform lists to dict with email as key and # number of (history/dataset)/size of history as value - histories = dict( [ ( _.email if _.email is not None else "Unknown", \ - int( _.history ) ) for _ in histories.execute() ] ) - datasets = dict( [ ( _.email if _.email is not None else "Unknown", \ - (int( _.dataset ), int( _.size )) ) for _ in datasets.execute () ] ) + histories = dict( [ ( _.email if _.email is not None else "Unknown", int( _.history ) ) + for _ in histories.execute() ] ) + datasets = dict( [ ( _.email if _.email is not None else "Unknown", (int( _.dataset ), int( _.size )) ) + for _ in datasets.execute() ] ) sorting_functions = [ - lambda first, second: descending if first[0].lower () > second[0].lower () else -descending, \ - lambda first, second: descending if histories.get(first, 0) < histories.get(second, 0) else -descending, \ - lambda first, second: descending if datasets.get(first, [0])[0] < datasets.get(second, [0])[0] else -descending, \ - lambda first, second: descending if datasets.get(first, [0,0])[1] < datasets.get(second, [0,0])[1] else -descending + lambda first, second: descending if first[0].lower() > second[0].lower() else -descending, + lambda first, second: descending if histories.get(first, 0) < histories.get(second, 0) else -descending, + lambda first, second: descending if datasets.get(first, [0])[0] < datasets.get(second, [0])[0] else -descending, + lambda first, second: descending if datasets.get(first, [0, 0])[1] < datasets.get(second, [0, 0])[1] else -descending ] - # fetch all users + # fetch all users users = list(set(histories.keys()) | set(datasets.keys())) # sort users depending on sort function, defined by user choices @@ -119,7 +116,7 @@ class History( BaseUIController ): for user in users: dataset = datasets.get(user, [0, 0]) history = histories.get(user, 0) - data[user] = ("%d (%s)"%( history, int_to_octet( dataset[1] ) ), dataset[0] ) + data[user] = ("%d (%s)" % ( history, int_to_octet( dataset[1] ) ), dataset[0] ) return trans.fill_template( '/webapps/reports/history_and_dataset_per_user.mako', data=data, @@ -135,37 +132,37 @@ class History( BaseUIController ): - the name of history - the number of dataset foreach type """ - + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 user_selection = kwd.get( 'user_selection', None) - + # select d.state, h.name # from dataset d, history h , history_dataset_association hda # where hda.history_id=h.id and hda.dataset_id=d.id order by h.state; from_obj = [ galaxy.model.Dataset.table, galaxy.model.History.table, galaxy.model.HistoryDatasetAssociation.table] if user_selection is not None: from_obj.append( galaxy.model.User.table ) - whereclause = and_( galaxy.model.Dataset.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,\ - galaxy.model.History.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.history_id, \ - galaxy.model.User.table.c.id==galaxy.model.History.table.c.user_id, \ - galaxy.model.User.table.c.email==user_selection ) - else : - whereclause = and_(galaxy.model.Dataset.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,\ - galaxy.model.History.table.c.id==galaxy.model.HistoryDatasetAssociation.table.c.history_id) + whereclause = and_( galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id, + galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id, + galaxy.model.User.table.c.id == galaxy.model.History.table.c.user_id, + galaxy.model.User.table.c.email == user_selection ) + else: + whereclause = and_(galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id, + galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id) histories = sa.select( ( galaxy.model.Dataset.table.c.state.label( 'state' ), - galaxy.model.History.table.c.name.label( 'name' ) ), - from_obj=from_obj, - whereclause=whereclause, - order_by = [ 'name' ] ) + galaxy.model.History.table.c.name.label( 'name' ) ), + from_obj=from_obj, + whereclause=whereclause, + order_by=[ 'name' ] ) # execute requests, replace None fields by "Unknown" - data = [ ( _.name if _.name is not None else "NoNamedHistory", \ - _.state ) for _ in histories.execute() ] + data = [ ( _.name if _.name is not None else "NoNamedHistory", _.state ) + for _ in histories.execute() ] # sort by names descending or ascending - data.sort(lambda first, second: descending if first[0].lower () > second[0].lower () else -descending) + data.sort(lambda first, second: descending if first[0].lower() > second[0].lower() else -descending) # fetch names in the first list and status in the second if data: @@ -173,13 +170,12 @@ class History( BaseUIController ): else: names, status = [], [] - - possible_status = {"ok" :0, "upload": 1, "paused": 2, "queued": 3, "error": 4, "discarded": 5} - number_of_possible_status = len( possible_status ) + 1 # + 1 to handle unknown status! + possible_status = {"ok": 0, "upload": 1, "paused": 2, "queued": 3, "error": 4, "discarded": 5} + number_of_possible_status = len( possible_status ) + 1 # + 1 to handle unknown status! # to keep ordered datas = collections.OrderedDict() - for no, name in enumerate (names): + for no, name in enumerate(names): if name not in datas: if user_cutoff > 0: if len( datas ) == user_cutoff: @@ -196,4 +192,4 @@ class History( BaseUIController ): data=datas, user_cutoff=user_cutoff, descending=descending, - message=message ) \ No newline at end of file + message=message ) From a66800465614042b75c0f30217b0c1d9075aaed1 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 10 May 2016 13:58:55 -0400 Subject: [PATCH 06/11] Slight bit of extra import cleanup in reports/controllers/history --- lib/galaxy/webapps/reports/controllers/history.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/history.py b/lib/galaxy/webapps/reports/controllers/history.py index f0757027bd6..3337a0db3c4 100644 --- a/lib/galaxy/webapps/reports/controllers/history.py +++ b/lib/galaxy/webapps/reports/controllers/history.py @@ -1,15 +1,13 @@ import collections -import sqlalchemy as sa import logging +import sqlalchemy as sa +from markupsafe import escape +from sqlalchemy import and_ import galaxy.model -import pkg_resources from galaxy import util -from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseUIController, web -from markupsafe import escape -pkg_resources.require( "SQLAlchemy >= 0.4" ) log = logging.getLogger( __name__ ) From 3107cfed9d502ea9baf0cab14a19cbf3c8bdec61 Mon Sep 17 00:00:00 2001 From: BalthazarPavot Date: Thu, 7 Jul 2016 17:32:52 +0200 Subject: [PATCH 07/11] Update tools.py --- .../webapps/reports/controllers/tools.py | 270 +++++++++--------- 1 file changed, 132 insertions(+), 138 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py index d985749ade6..a97ca30c2ed 100644 --- a/lib/galaxy/webapps/reports/controllers/tools.py +++ b/lib/galaxy/webapps/reports/controllers/tools.py @@ -1,16 +1,13 @@ import collections -import calendar, operator import sqlalchemy as sa import logging -from datetime import datetime, date, timedelta -from time import strftime -import collections +from datetime import timedelta import galaxy.model import pkg_resources from galaxy import util -from galaxy.model.orm import and_, not_, or_ +from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseUIController, web from markupsafe import escape @@ -18,9 +15,9 @@ pkg_resources.require( "SQLAlchemy >= 0.4" ) log = logging.getLogger( __name__ ) -def int_to_octet (size): +def int_to_octet(size): try: - size = float (size) + size = float(size) except ValueError: return "???" except TypeError: @@ -33,9 +30,10 @@ def int_to_octet (size): size /= 1000. no_unit += 1 try: - return "%.2f %s"%( size, units[no_unit] ) + return "%.2f %s" % ( size, units[no_unit] ) except IndexError: - return "%.0f %s"%( size * ( ( no_unit-len( units )+1 ) * 1000. ), units[-1] ) + return "%.0f %s" % ( size * ( ( no_unit - len( units ) + 1 ) * 1000. ), units[-1] ) + class Tools( BaseUIController ): """ @@ -45,15 +43,15 @@ class Tools( BaseUIController ): the "href" dict, in data.mako (templates/webapps/reports). """ - def formated (self, date, colored = False): - splited = str (date).split(',') - if len (splited) == 2: - returned = "%s %dH" % (splited[0], int (splited[1].split (':')[0])) + def formated(self, date, colored=False): + splited = str(date).split(',') + if len(splited) == 2: + returned = "%s %dH" % (splited[0], int(splited[1].split(':')[0])) if colored: return '' + returned + '' return returned else: - splited = tuple ([float (_) for _ in str (date).split (':')]) + splited = tuple([float(_) for _ in str(date).split(':')]) if splited[0]: returned = '%d h. %d min.' % splited[:2] if colored: @@ -66,51 +64,50 @@ class Tools( BaseUIController ): @web.expose def tools_and_job_state( self, trans, **kwd ): """ - fill tools_and_job_state_peer_month.mako template with + fill tools_and_job_state.mako template with - the name of the tool - the number of jobs using this tool in state 'ok' - the number of jobs using this tool in error """ - + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) - + # sort by history space, or by user mail or by number of history/dataset sort_by = kwd.get( 'sorting', 'Tool' ) sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2 descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 - sort_functions = (lambda first, second: descending if first.lower () > second.lower () else -descending, \ - lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > \ - tools_and_jobs_ok.get( second ) else descending, \ - lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > \ + sort_functions = (lambda first, second: descending if first.lower() > second.lower() else -descending, + lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > + tools_and_jobs_ok.get( second ) else descending, + lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > tools_and_jobs_error.get( second, 0 ) else descending) - data = collections.OrderedDict() # select count(id), tool_id from job where state='ok' group by tool_id; tools_and_jobs_ok = sa.select( (galaxy.model.Job.table.c.tool_id .label( 'tool' ), sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), from_obj=[ galaxy.model.Job.table], - whereclause=(galaxy.model.Job.table.c.state=='ok'), + whereclause=(galaxy.model.Job.table.c.state == 'ok'), group_by=[ 'tool' ] ) # select count(id), tool_id from job where state='error' group by tool_id; tools_and_jobs_error = sa.select( (galaxy.model.Job.table.c.tool_id .label( 'tool' ), sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), from_obj=[ galaxy.model.Job.table], - whereclause=(galaxy.model.Job.table.c.state=='error'), + whereclause=(galaxy.model.Job.table.c.state == 'error'), group_by=[ 'tool' ] ) - tools_and_jobs_ok = dict( list( tools_and_jobs_ok.execute () ) ) - tools_and_jobs_error = dict( list( tools_and_jobs_error.execute () ) ) + tools_and_jobs_ok = dict( list( tools_and_jobs_ok.execute() ) ) + tools_and_jobs_error = dict( list( tools_and_jobs_error.execute() ) ) # select each job name one time - tools = list (set(tools_and_jobs_ok.keys ()) | set (tools_and_jobs_error.keys ())) - tools.sort (sort_functions[ sorting ]) + tools = list(set(tools_and_jobs_ok.keys()) | set(tools_and_jobs_error.keys())) + tools.sort(sort_functions[ sorting ]) for tool in tools: - data[tool] = (str( tools_and_jobs_ok.get (tool, '-') ), str( tools_and_jobs_error.get (tool, '-') ) ) + data[tool] = (str( tools_and_jobs_ok.get(tool, '-') ), str( tools_and_jobs_error.get(tool, '-') ) ) return trans.fill_template( '/webapps/reports/tools_and_job_state.mako', data=data, @@ -120,31 +117,25 @@ class Tools( BaseUIController ): message=message ) @web.expose - def tools_and_job_state_peer_month (self, trans, **kwd ): + def tools_and_job_state_per_month(self, trans, **kwd ): """ - fill tools_and_job_state_peer_month.mako template with + fill tools_and_job_state_per_month.mako template with - the name of the tool - the number of jobs using this tool in state 'ok' - the number of jobs using this tool in error """ - + message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) - + # sort by history space, or by user mail or by number of history/dataset - sort_by = kwd.get( 'sorting', 'Tool' ) - sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2 - descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 + # sort_by = kwd.get( 'sorting', 'Tool' ) + # sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2 + # descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 tool = kwd.get( 'tool', None) if tool is None: - raise TypeError ("Tool can't be None") - - sort_functions = (lambda first, second: descending if first.lower () > second.lower () else -descending, \ - lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > \ - tools_and_jobs_ok.get( second ) else descending, \ - lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > \ - tools_and_jobs_error.get( second, 0 ) else descending) + raise TypeError("Tool can't be None") data = collections.OrderedDict() @@ -152,69 +143,75 @@ class Tools( BaseUIController ): date_and_jobs_ok = sa.select( (sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), from_obj=[ galaxy.model.Job.table], - whereclause=and_( galaxy.model.Job.table.c.state=='ok', - galaxy.model.Job.table.c.tool_id==tool ), + whereclause=and_( galaxy.model.Job.table.c.state == 'ok', + galaxy.model.Job.table.c.tool_id == tool ), group_by=[ 'date' ] ) # select count(id), create_time from job where state='error' and tool_id=$tool group by date; date_and_jobs_error = sa.select( (sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), from_obj=[ galaxy.model.Job.table], - whereclause=and_( galaxy.model.Job.table.c.state=='error', - galaxy.model.Job.table.c.tool_id==tool ), + whereclause=and_( galaxy.model.Job.table.c.state == 'error', + galaxy.model.Job.table.c.tool_id == tool ), group_by=[ 'date' ] ) - date_and_jobs_ok = dict( list( date_and_jobs_ok.execute () ) ) - date_and_jobs_error = dict( list( date_and_jobs_error.execute () ) ) + # sort_functions = (lambda first, second: descending if first.lower() > second.lower() else -descending, + # lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > + # tools_and_jobs_ok.get( second ) else descending, + # lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > + # tools_and_jobs_error.get( second, 0 ) else descending) + + date_and_jobs_ok = dict( list( date_and_jobs_ok.execute() ) ) + date_and_jobs_error = dict( list( date_and_jobs_error.execute() ) ) # select each date - dates = list (set(date_and_jobs_ok.keys ()) | set (date_and_jobs_error.keys ())) - dates.sort (reverse=True) + dates = list(set(date_and_jobs_ok.keys()) | set(date_and_jobs_error.keys())) + dates.sort(reverse=True) for date in dates: date_key = date.strftime( "%B %Y" ) if date_key not in data: - data[date_key] = [int( date_and_jobs_ok.get (date, 0) ), int( date_and_jobs_error.get (date, 0) ) ] + data[date_key] = [int( date_and_jobs_ok.get(date, 0) ), int( date_and_jobs_error.get(date, 0) ) ] else : - data[date_key][0] += int( date_and_jobs_ok.get ( date, 0 ) ) + data[date_key][0] += int( date_and_jobs_ok.get( date, 0 ) ) data[date_key][1] += int( date_and_jobs_error.get( date, 0 ) ) - - return trans.fill_template( '/webapps/reports/tools_and_job_state_peer_month.mako', + return trans.fill_template( '/webapps/reports/tools_and_job_state_per_month.mako', data=data, tool=tool, user_cutoff=user_cutoff, message=message ) @web.expose - def tool_execution_time (self, trans, **kwd): + def tool_execution_time(self, trans, **kwd): """ - Fill th template tool_execution_time.mako with informations: + Fill the template tool_execution_time.mako with informations: - Tool name - Tool average execution time - last job execution time - min and max execution time """ - #~ liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois (?) - user_cutoff = int (kwd.get ("user_cutoff", 60)) - sort_by = kwd.get ("sort_by", "tool") + + # liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois (?) + user_cutoff = int(kwd.get("user_cutoff", 60)) + sort_by = kwd.get("sort_by", "tool") descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 sort_by = 0 if sort_by == "tool" else 1 if sort_by == "avg" else 2 if sort_by == "min" else 3 - color = True if kwd.get ("color", '') == "True" else False + color = True if kwd.get("color", '') == "True" else False data = {} ordered_data = collections.OrderedDict() field_sort = lambda first, second, field: descending if data[first][field] < data[second][field] else -descending sort_functions = [ - lambda first, second: -descending if first.lower () < second.lower () else descending, - lambda first, second: field_sort (first, second, "avg"), - lambda first, second: field_sort (first, second, "min"), - lambda first, second: field_sort (first, second, "max")] + lambda first, second: -descending if first.lower() < second.lower() else descending, + lambda first, second: field_sort(first, second, "avg"), + lambda first, second: field_sort(first, second, "min"), + lambda first, second: field_sort(first, second, "max")] - jobs_times = sa.select ( (galaxy.model.Job.table.c.tool_id.label( "name" ), - galaxy.model.Job.table.c.create_time.label ("create_time"), - galaxy.model.Job.table.c.update_time.label ("update_time"), + jobs_times = sa.select( (galaxy.model.Job.table.c.tool_id.label( "name" ), + galaxy.model.Job.table.c.create_time.label("create_time"), + galaxy.model.Job.table.c.update_time.label("update_time"), galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - from_obj = [galaxy.model.Job.table]) + from_obj=[galaxy.model.Job.table]) jobs_times = [(name, (create, update, time)) for name, create, update, time in jobs_times.execute()] for tool, attr in jobs_times: @@ -222,26 +219,25 @@ class Tools( BaseUIController ): data[tool] = {"last": [(attr[1], attr[0])], "avg": [attr[2]]} else: - data[tool]["last"].append ((attr[1], attr[0])) - data[tool]["avg"].append (attr[2]) + data[tool]["last"].append((attr[1], attr[0])) + data[tool]["avg"].append(attr[2]) for tool in data: - data[tool]["min"] = min (data[tool]["avg"]) - data[tool]["max"] = max (data[tool]["avg"]) - last = max (data[tool]["last"]) + data[tool]["min"] = min(data[tool]["avg"]) + data[tool]["max"] = max(data[tool]["avg"]) + last = max(data[tool]["last"]) data[tool]["last"] = last[0] - last[1] - data[tool]["avg"] = sum (data[tool]["avg"], timedelta()) / len (data[tool]["avg"]) + data[tool]["avg"] = sum(data[tool]["avg"], timedelta()) / len(data[tool]["avg"]) - - tools = data.keys () + tools = data.keys() if user_cutoff: tools = tools[:user_cutoff] - tools.sort (sort_functions[sort_by]) + tools.sort(sort_functions[sort_by]) for tool in tools: - ordered_data[tool] = {"min": self.formated (data[tool]["min"], color), - "max": self.formated (data[tool]["max"], color), - "avg": self.formated (data[tool]["avg"], color), - "last": self.formated (data[tool]["last"], color)} + ordered_data[tool] = {"min": self.formated(data[tool]["min"], color), + "max": self.formated(data[tool]["max"], color), + "avg": self.formated(data[tool]["avg"], color), + "last": self.formated(data[tool]["last"], color)} return trans.fill_template( '/webapps/reports/tool_execution_time.mako', data=ordered_data, @@ -250,46 +246,47 @@ class Tools( BaseUIController ): sort_by=sort_by ) @web.expose - def tool_execution_time_peer_month (self, trans, **kwd): + def tool_execution_time_per_month(self, trans, **kwd): """ - Fill th template tool_execution_time_peer_month.mako with informations: + Fill the template tool_execution_time_per_month.mako with informations: - Tool average execution time - last job execution time - min and max execution time """ - #~ liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois (?) - user_cutoff = int (kwd.get ("user_cutoff", 60)) - sort_by = kwd.get ("sort_by", "month") + + # liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois(?) + user_cutoff = int(kwd.get("user_cutoff", 60)) + sort_by = kwd.get("sort_by", "month") descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 sort_by = 0 if sort_by == "month" else 1 if sort_by == "min" else 2 if sort_by == "max" else 3 - tool = kwd.get ("tool", None) - color = True if kwd.get ("color", '') == "True" else False - + tool = kwd.get("tool", None) + color = True if kwd.get("color", '') == "True" else False + if tool is None: - raise ValueError ("Tool can't be None") + raise ValueError("Tool can't be None") ordered_data = collections.OrderedDict() sort_functions = [(lambda first, second, i=i: descending if first[i] < second[i] else -descending) for i in range(4)] - jobs_times = sa.select ((sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time ).label ('date'), - sa.func.max (galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - sa.func.avg (galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - sa.func.min (galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time)), - from_obj = [galaxy.model.Job.table], - whereclause = galaxy.model.Job.table.c.tool_id == tool, - group_by = ['date']) + jobs_times = sa.select((sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time ).label('date'), + sa.func.max(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + sa.func.avg(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + sa.func.min(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time)), + from_obj=[galaxy.model.Job.table], + whereclause=galaxy.model.Job.table.c.tool_id == tool, + group_by=['date']) - months = list (jobs_times.execute()) - months.sort (sort_functions[sort_by]) + months = list(jobs_times.execute()) + months.sort(sort_functions[sort_by]) if user_cutoff: months = months[:user_cutoff] for month in months: - ordered_data[str(month[0]).split(' ')[0][:-3]] = (self.formated (month[1], color), - self.formated (month[2], color), - self.formated (month[3], color)) + ordered_data[str(month[0]).split(' ')[0][:-3]] = (self.formated(month[1], color), + self.formated(month[2], color), + self.formated(month[3], color)) - return trans.fill_template( '/webapps/reports/tool_execution_time_peer_month.mako', + return trans.fill_template( '/webapps/reports/tool_execution_time_per_month.mako', data=ordered_data, tool=tool, descending=descending, @@ -297,35 +294,35 @@ class Tools( BaseUIController ): sort_by=sort_by ) @web.expose - def tool_error_messages (self, trans, **kwd): - tool_name = kwd.get ("tool", None) - descending = 1 if kwd.get ("descending", 'desc') == "desc" else -1 - sort_by = 0 if kwd.get ("sort_by", "time") == "time" else 1 - cutoff = int (kwd.get ("user_cutoff", 60)) - sort_functions = (lambda _a, _b: -descending if counter[_a][1] > counter[_b][1] else descending, + def tool_error_messages(self, trans, **kwd): + tool_name = kwd.get("tool", None) + descending = 1 if kwd.get("descending", 'desc') == "desc" else -1 + sort_by = 0 if kwd.get("sort_by", "time") == "time" else 1 + cutoff = int(kwd.get("user_cutoff", 60)) + sort_functions = (lambda _a, _b: -descending if counter[_a][1] > counter[_b][1] else descending, lambda _a, _b: -descending if counter[_a][0] > counter[_b][0] else descending) if tool_name is None: - raise ValueError ("Tool can't be none") - tool_errors = [[unicode (a), b] for a, b in \ - sa.select ((galaxy.model.Job.table.c.stderr, galaxy.model.Job.table.c.create_time), - from_obj = [galaxy.model.Job.table], - whereclause=and_(galaxy.model.Job.table.c.tool_id==tool_name, - galaxy.model.Job.table.c.state=='error')).execute ()] + raise ValueError("Tool can't be none") + tool_errors = [[unicode(a), b] for a, b in + sa.select((galaxy.model.Job.table.c.stderr, galaxy.model.Job.table.c.create_time), + from_obj=[galaxy.model.Job.table], + whereclause=and_(galaxy.model.Job.table.c.tool_id == tool_name, + galaxy.model.Job.table.c.state == 'error')).execute()] counter = {} for error in tool_errors: try: - error[0] = unicode(error[0].decode ("utf-8")) + error[0] = unicode(error[0].decode("utf-8")) # encoding tested: # latin-1 ; iso-8859-1 ; alien ; cenc ; cp037 ; cp437 ; base64 ; utf-8 ; utf-16 ; ascii ; hex except UnicodeEncodeError: - for no, lettre in enumerate (error[0]): + for no, lettre in enumerate(error[0]): try: - str (lettre.decode ("utf-8")) + str(lettre.decode("utf-8")) except UnicodeEncodeError: try: - error[0] = error[0].replace (error[0][no], '?') + error[0] = error[0].replace(error[0][no], '?') except UnicodeEncodeError: error[0] = "This error contains special character and can't be displayed." break @@ -334,38 +331,35 @@ class Tools( BaseUIController ): else: counter[error[0]] = [1, error[1]] - data = collections.OrderedDict () - keys = counter.keys () + data = collections.OrderedDict() + keys = counter.keys() if cutoff: keys = keys[:cutoff] - keys.sort (sort_functions [sort_by]) + keys.sort(sort_functions[sort_by]) spaces = [' ', '\t', ' '] for key in keys: - new_key = '
'.join ([_ for _ in key.split ('\n') if _ and _ not in spaces]) - if len (new_key) >= 100: + new_key = '
'.join([_ for _ in key.split('\n') if _ and _ not in spaces]) + if len(new_key) >= 100: to_replace = [] - words = key.split ('\n') + words = key.split('\n') for word in words: if word in to_replace: continue - if words.count (word) > 1: + if words.count(word) > 1: print word - to_replace.append (word) + to_replace.append(word) for word in to_replace: - sentence = ("
"+word) * 2 + sentence = ("
" + word) * 2 count = 2 - while sentence+"
"+word in new_key: - sentence += "
"+word + while sentence + "
" + word in new_key: + sentence += "
" + word count += 1 print sentence, count if sentence in new_key: - new_key = new_key.replace (sentence, '
' + word + " [this line in %d times]"%(count)) + new_key = new_key.replace(sentence, '
' + word + " [this line in %d times]" % (count)) data[new_key] = counter[key] - return trans.fill_template ("/webapps/reports/tool_error_messages.mako", - data = data, - descending = descending, - tool_name=tool_name, - sort_by=sort_by, - user_cutoff=cutoff) \ No newline at end of file + return trans.fill_template("/webapps/reports/tool_error_messages.mako", + data=data, descending=descending, tool_name=tool_name, sort_by=sort_by, + user_cutoff=cutoff) From 12c65d4a6365e34660c78dab7bb43b2cf292eb61 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 26 Jul 2016 12:24:54 -0400 Subject: [PATCH 08/11] fix import of and_ operator --- lib/galaxy/webapps/reports/controllers/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py index a97ca30c2ed..57c4521a62d 100644 --- a/lib/galaxy/webapps/reports/controllers/tools.py +++ b/lib/galaxy/webapps/reports/controllers/tools.py @@ -7,7 +7,7 @@ from datetime import timedelta import galaxy.model import pkg_resources from galaxy import util -from galaxy.model.orm import and_ +from sqlalchemy import and_ from galaxy.web.base.controller import BaseUIController, web from markupsafe import escape From 28a990470738ac821a6a2851306dac9723767c12 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 26 Jul 2016 13:06:18 -0400 Subject: [PATCH 09/11] fix menu items to actually display new templates reword certain texts reorder imports and fix style remove unnecessary imports --- .../webapps/reports/controllers/history.py | 13 +- .../webapps/reports/controllers/tools.py | 133 +++++++++--------- .../webapps/reports/controllers/users.py | 24 ++-- .../reports/history_and_dataset_per_user.mako | 6 +- .../reports/history_and_dataset_type.mako | 4 +- .../webapps/reports/history_per_user.mako | 6 +- templates/webapps/reports/index.mako | 36 ++--- .../webapps/reports/tool_execution_time.mako | 8 +- ...ako => tool_execution_time_per_month.mako} | 8 +- .../webapps/reports/tools_and_job_state.mako | 8 +- ...ako => tools_and_job_state_per_month.mako} | 2 +- 11 files changed, 122 insertions(+), 126 deletions(-) rename templates/webapps/reports/{tool_execution_time_peer_month.mako => tool_execution_time_per_month.mako} (94%) rename templates/webapps/reports/{tools_and_job_state_peer_month.mako => tools_and_job_state_per_month.mako} (93%) diff --git a/lib/galaxy/webapps/reports/controllers/history.py b/lib/galaxy/webapps/reports/controllers/history.py index 3337a0db3c4..7a75c6a2ba8 100644 --- a/lib/galaxy/webapps/reports/controllers/history.py +++ b/lib/galaxy/webapps/reports/controllers/history.py @@ -1,13 +1,14 @@ +import galaxy.model import collections import logging import sqlalchemy as sa -from markupsafe import escape -from sqlalchemy import and_ -import galaxy.model from galaxy import util from galaxy.web.base.controller import BaseUIController, web +from markupsafe import escape +from sqlalchemy import and_ + log = logging.getLogger( __name__ ) @@ -34,11 +35,10 @@ def int_to_octet(size): class History( BaseUIController ): """ Class defining functions used by reports to make requests to get - informations and fill templates before beeing displayed. + informations and fill templates before being displayed. The name of function must be the same as as the field "action" of - the "href" dict, in data.mako (templates/webapps/reports). + the "href" dict, in .mako templates (templates/webapps/reports). """ - @web.expose def history_and_dataset_per_user( self, trans, **kwd ): """ @@ -130,7 +130,6 @@ class History( BaseUIController ): - the name of history - the number of dataset foreach type """ - message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py index 57c4521a62d..cf11c968b41 100644 --- a/lib/galaxy/webapps/reports/controllers/tools.py +++ b/lib/galaxy/webapps/reports/controllers/tools.py @@ -1,17 +1,16 @@ import collections -import sqlalchemy as sa import logging -from datetime import timedelta - import galaxy.model -import pkg_resources +import sqlalchemy as sa + from galaxy import util -from sqlalchemy import and_ from galaxy.web.base.controller import BaseUIController, web + +from sqlalchemy import and_ +from datetime import timedelta from markupsafe import escape -pkg_resources.require( "SQLAlchemy >= 0.4" ) log = logging.getLogger( __name__ ) @@ -38,12 +37,12 @@ def int_to_octet(size): class Tools( BaseUIController ): """ Class defining functions used by reports to make requests to get - informations and fill templates before beeing displayed. + informations and fill templates before being displayed. The name of function must be the same as as the field "action" of - the "href" dict, in data.mako (templates/webapps/reports). + the "href" dict, in .mako templates (templates/webapps/reports). """ - def formated(self, date, colored=False): + def formatted(self, date, colored=False): splited = str(date).split(',') if len(splited) == 2: returned = "%s %dH" % (splited[0], int(splited[1].split(':')[0])) @@ -77,27 +76,25 @@ class Tools( BaseUIController ): sort_by = kwd.get( 'sorting', 'Tool' ) sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2 descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 - sort_functions = (lambda first, second: descending if first.lower() > second.lower() else -descending, - lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > - tools_and_jobs_ok.get( second ) else descending, - lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > - tools_and_jobs_error.get( second, 0 ) else descending) + sort_functions = ( lambda first, second: descending if first.lower() > second.lower() else -descending, + lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > tools_and_jobs_ok.get( second ) else descending, + lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > tools_and_jobs_error.get( second, 0 ) else descending ) data = collections.OrderedDict() # select count(id), tool_id from job where state='ok' group by tool_id; - tools_and_jobs_ok = sa.select( (galaxy.model.Job.table.c.tool_id .label( 'tool' ), - sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), - from_obj=[ galaxy.model.Job.table], - whereclause=(galaxy.model.Job.table.c.state == 'ok'), - group_by=[ 'tool' ] ) + tools_and_jobs_ok = sa.select( ( galaxy.model.Job.table.c.tool_id .label( 'tool' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=(galaxy.model.Job.table.c.state == 'ok'), + group_by=[ 'tool' ] ) # select count(id), tool_id from job where state='error' group by tool_id; - tools_and_jobs_error = sa.select( (galaxy.model.Job.table.c.tool_id .label( 'tool' ), - sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), - from_obj=[ galaxy.model.Job.table], - whereclause=(galaxy.model.Job.table.c.state == 'error'), - group_by=[ 'tool' ] ) + tools_and_jobs_error = sa.select( ( galaxy.model.Job.table.c.tool_id .label( 'tool' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=(galaxy.model.Job.table.c.state == 'error'), + group_by=[ 'tool' ] ) tools_and_jobs_ok = dict( list( tools_and_jobs_ok.execute() ) ) tools_and_jobs_error = dict( list( tools_and_jobs_error.execute() ) ) @@ -140,20 +137,18 @@ class Tools( BaseUIController ): data = collections.OrderedDict() # select count(id), create_time from job where state='ok' and tool_id=$tool group by date; - date_and_jobs_ok = sa.select( (sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), - sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), - from_obj=[ galaxy.model.Job.table], - whereclause=and_( galaxy.model.Job.table.c.state == 'ok', - galaxy.model.Job.table.c.tool_id == tool ), - group_by=[ 'date' ] ) + date_and_jobs_ok = sa.select( ( sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=and_( galaxy.model.Job.table.c.state == 'ok', galaxy.model.Job.table.c.tool_id == tool ), + group_by=[ 'date' ] ) # select count(id), create_time from job where state='error' and tool_id=$tool group by date; - date_and_jobs_error = sa.select( (sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), - sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), - from_obj=[ galaxy.model.Job.table], - whereclause=and_( galaxy.model.Job.table.c.state == 'error', - galaxy.model.Job.table.c.tool_id == tool ), - group_by=[ 'date' ] ) + date_and_jobs_error = sa.select( ( sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ), + sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ), + from_obj=[ galaxy.model.Job.table], + whereclause=and_( galaxy.model.Job.table.c.state == 'error', galaxy.model.Job.table.c.tool_id == tool ), + group_by=[ 'date' ] ) # sort_functions = (lambda first, second: descending if first.lower() > second.lower() else -descending, # lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > @@ -171,7 +166,7 @@ class Tools( BaseUIController ): date_key = date.strftime( "%B %Y" ) if date_key not in data: data[date_key] = [int( date_and_jobs_ok.get(date, 0) ), int( date_and_jobs_error.get(date, 0) ) ] - else : + else: data[date_key][0] += int( date_and_jobs_ok.get( date, 0 ) ) data[date_key][1] += int( date_and_jobs_error.get( date, 0 ) ) @@ -207,17 +202,16 @@ class Tools( BaseUIController ): lambda first, second: field_sort(first, second, "min"), lambda first, second: field_sort(first, second, "max")] - jobs_times = sa.select( (galaxy.model.Job.table.c.tool_id.label( "name" ), - galaxy.model.Job.table.c.create_time.label("create_time"), - galaxy.model.Job.table.c.update_time.label("update_time"), - galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - from_obj=[galaxy.model.Job.table]) + jobs_times = sa.select( ( galaxy.model.Job.table.c.tool_id.label( "name" ), + galaxy.model.Job.table.c.create_time.label("create_time"), + galaxy.model.Job.table.c.update_time.label("update_time"), + galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time ), + from_obj=[galaxy.model.Job.table]) jobs_times = [(name, (create, update, time)) for name, create, update, time in jobs_times.execute()] for tool, attr in jobs_times: if tool not in data: - data[tool] = {"last": [(attr[1], attr[0])], - "avg": [attr[2]]} + data[tool] = { "last": [(attr[1], attr[0])], "avg": [attr[2]] } else: data[tool]["last"].append((attr[1], attr[0])) data[tool]["avg"].append(attr[2]) @@ -234,10 +228,10 @@ class Tools( BaseUIController ): tools = tools[:user_cutoff] tools.sort(sort_functions[sort_by]) for tool in tools: - ordered_data[tool] = {"min": self.formated(data[tool]["min"], color), - "max": self.formated(data[tool]["max"], color), - "avg": self.formated(data[tool]["avg"], color), - "last": self.formated(data[tool]["last"], color)} + ordered_data[tool] = { "min": self.formatted(data[tool]["min"], color), + "max": self.formatted(data[tool]["max"], color), + "avg": self.formatted(data[tool]["avg"], color), + "last": self.formatted(data[tool]["last"], color) } return trans.fill_template( '/webapps/reports/tool_execution_time.mako', data=ordered_data, @@ -268,13 +262,13 @@ class Tools( BaseUIController ): ordered_data = collections.OrderedDict() sort_functions = [(lambda first, second, i=i: descending if first[i] < second[i] else -descending) for i in range(4)] - jobs_times = sa.select((sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time ).label('date'), - sa.func.max(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - sa.func.avg(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), - sa.func.min(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time)), - from_obj=[galaxy.model.Job.table], - whereclause=galaxy.model.Job.table.c.tool_id == tool, - group_by=['date']) + jobs_times = sa.select( ( sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time ).label('date'), + sa.func.max(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + sa.func.avg(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time), + sa.func.min(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time) ), + from_obj=[galaxy.model.Job.table], + whereclause=galaxy.model.Job.table.c.tool_id == tool, + group_by=['date'] ) months = list(jobs_times.execute()) months.sort(sort_functions[sort_by]) @@ -282,9 +276,9 @@ class Tools( BaseUIController ): months = months[:user_cutoff] for month in months: - ordered_data[str(month[0]).split(' ')[0][:-3]] = (self.formated(month[1], color), - self.formated(month[2], color), - self.formated(month[3], color)) + ordered_data[str(month[0]).split(' ')[0][:-3]] = ( self.formatted(month[1], color), + self.formatted(month[2], color), + self.formatted(month[3], color) ) return trans.fill_template( '/webapps/reports/tool_execution_time_per_month.mako', data=ordered_data, @@ -299,16 +293,16 @@ class Tools( BaseUIController ): descending = 1 if kwd.get("descending", 'desc') == "desc" else -1 sort_by = 0 if kwd.get("sort_by", "time") == "time" else 1 cutoff = int(kwd.get("user_cutoff", 60)) - sort_functions = (lambda _a, _b: -descending if counter[_a][1] > counter[_b][1] else descending, - lambda _a, _b: -descending if counter[_a][0] > counter[_b][0] else descending) + sort_functions = ( lambda _a, _b: -descending if counter[_a][1] > counter[_b][1] else descending, + lambda _a, _b: -descending if counter[_a][0] > counter[_b][0] else descending ) if tool_name is None: raise ValueError("Tool can't be none") - tool_errors = [[unicode(a), b] for a, b in - sa.select((galaxy.model.Job.table.c.stderr, galaxy.model.Job.table.c.create_time), - from_obj=[galaxy.model.Job.table], - whereclause=and_(galaxy.model.Job.table.c.tool_id == tool_name, - galaxy.model.Job.table.c.state == 'error')).execute()] + tool_errors = [ [unicode(a), b] for a, b in + sa.select((galaxy.model.Job.table.c.stderr, galaxy.model.Job.table.c.create_time), + from_obj=[galaxy.model.Job.table], + whereclause=and_( galaxy.model.Job.table.c.tool_id == tool_name, + galaxy.model.Job.table.c.state == 'error')).execute() ] counter = {} for error in tool_errors: @@ -360,6 +354,9 @@ class Tools( BaseUIController ): new_key = new_key.replace(sentence, '
' + word + " [this line in %d times]" % (count)) data[new_key] = counter[key] - return trans.fill_template("/webapps/reports/tool_error_messages.mako", - data=data, descending=descending, tool_name=tool_name, sort_by=sort_by, - user_cutoff=cutoff) + return trans.fill_template( "/webapps/reports/tool_error_messages.mako", + data=data, + descending=descending, + tool_name=tool_name, + sort_by=sort_by, + user_cutoff=cutoff ) diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index c28f221ea01..b842637fa6f 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -1,18 +1,18 @@ import calendar import logging import operator -from datetime import datetime, date, timedelta - -from markupsafe import escape -import sqlalchemy as sa -from sqlalchemy import false - import galaxy.model +import sqlalchemy as sa + from galaxy import util from galaxy.web.base.controller import BaseUIController, web from galaxy.webapps.reports.controllers.jobs import sorter from galaxy.webapps.reports.controllers.query import ReportQueryBuilder +from datetime import datetime, date, timedelta +from markupsafe import escape +from sqlalchemy import false + log = logging.getLogger( __name__ ) @@ -187,19 +187,19 @@ class Users( BaseUIController, ReportQueryBuilder ): @web.expose def history_per_user( self, trans, **kwd ): - message = escape( util.restore_text( kwd.get( 'message', '' ) ) ) user_cutoff = int( kwd.get( 'user_cutoff', 60 ) ) sorting = 0 if kwd.get( 'sorting', 'User' ) == 'User' else 1 descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1 sorting_functions = [ - lambda first, second: descending if first[0].lower () > second[0].lower () else -descending, \ - lambda first, second: descending if first[1] < second[1] else -descending] + lambda first, second: descending if first[0].lower() > second[0].lower() else -descending, + lambda first, second: descending if first[1] < second[1] else -descending ] - req = sa.select( ( sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ), - galaxy.model.User.table.c.username.label( 'username' ) ), + req = sa.select( + ( sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ), + galaxy.model.User.table.c.username.label( 'username' ) ), from_obj=[ sa.outerjoin( galaxy.model.History.table, galaxy.model.User.table ) ], - whereclause=galaxy.model.History.table.c.user_id==galaxy.model.User.table.c.id, + whereclause=galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id, group_by=[ 'username' ], order_by=[ sa.desc( 'username' ), 'history' ] ) diff --git a/templates/webapps/reports/history_and_dataset_per_user.mako b/templates/webapps/reports/history_and_dataset_per_user.mako index dfe83016502..da36d43ad19 100644 --- a/templates/webapps/reports/history_and_dataset_per_user.mako +++ b/templates/webapps/reports/history_and_dataset_per_user.mako @@ -6,14 +6,14 @@ %endif
-

Per-user number of History and Datasets

-

Listed in +

Histories and Datasets per User

+

Listed in %if descending == 1: descending %else: ascending %endif - order by + order by %if sorting == 0: Users %elif sorting == 1: diff --git a/templates/webapps/reports/history_and_dataset_type.mako b/templates/webapps/reports/history_and_dataset_type.mako index 2d80246b604..87e174b03c5 100644 --- a/templates/webapps/reports/history_and_dataset_type.mako +++ b/templates/webapps/reports/history_and_dataset_type.mako @@ -6,8 +6,8 @@ %endif
-

History and Datasets types

-

Listed in +

States of Datasets per History

+

Listed in %if descending == 1: descending %else: diff --git a/templates/webapps/reports/history_per_user.mako b/templates/webapps/reports/history_per_user.mako index d3703437c4e..76fa3443f3f 100644 --- a/templates/webapps/reports/history_per_user.mako +++ b/templates/webapps/reports/history_per_user.mako @@ -6,14 +6,14 @@ %endif
-

Per-user number of History

-

Listed in +

Histories per User

+

Listed in %if descending == 1: descending %else: ascending %endif - order by + order by %if sorting == 0: Users %else: diff --git a/templates/webapps/reports/index.mako b/templates/webapps/reports/index.mako index 4ae602fde99..1baf5a883f8 100644 --- a/templates/webapps/reports/index.mako +++ b/templates/webapps/reports/index.mako @@ -9,7 +9,7 @@ <%def name="stylesheets()"> - ${parent.stylesheets()} + ${parent.stylesheets()} ## Include "base.css" for styling tool menu and forms (details) ${h.css( "base", "autocomplete_tagging", "tool_menu" )} @@ -63,23 +63,12 @@
- Sample Tracking + Histories
-
Sequencing requests per month
-
Sequencing requests per user
-
-
-
-
-
- History -
-
-
-
History and dataset per user
-
History and dataset type
+
Histories and Datasets per User
+
States of Datasets per History
@@ -89,8 +78,8 @@

-
History and dataset per user
-
History and dataset type
+
States of Jobs per Tool
+
Execution Time per Tool
@@ -128,8 +117,19 @@
Disk space maintenance

+
+
+
+ Sample Tracking +
+
+
+
Sequencing requests per month
+
Sequencing requests per user
+
+
- + diff --git a/templates/webapps/reports/tool_execution_time.mako b/templates/webapps/reports/tool_execution_time.mako index a2d797a2336..18376da8102 100644 --- a/templates/webapps/reports/tool_execution_time.mako +++ b/templates/webapps/reports/tool_execution_time.mako @@ -3,14 +3,14 @@
-

Time execution peer tool

-

Listed in +

Execution Time per Tool

+

Listed in %if descending == 1: descending %else: ascending %endif - order by + order by %if sort_by == 0: Tool %elif sort_by == 1: @@ -64,7 +64,7 @@ %endif - ${tool} + ${tool} ${data[tool]["avg"]} ${data[tool]["last"]} diff --git a/templates/webapps/reports/tool_execution_time_peer_month.mako b/templates/webapps/reports/tool_execution_time_per_month.mako similarity index 94% rename from templates/webapps/reports/tool_execution_time_peer_month.mako rename to templates/webapps/reports/tool_execution_time_per_month.mako index 5ade9ec8c04..636b826a083 100644 --- a/templates/webapps/reports/tool_execution_time_peer_month.mako +++ b/templates/webapps/reports/tool_execution_time_per_month.mako @@ -3,14 +3,14 @@
-

Time execution for ${tool} peer month

-

Listed in +

Execution Time for ${tool} per Month

+

Listed in %if descending == 1: descending %else: ascending %endif - order by + order by %if sort_by == 0: Month %elif sort_by == 1: @@ -24,7 +24,7 @@
-
+

Top shown (0 = all).
diff --git a/templates/webapps/reports/tools_and_job_state.mako b/templates/webapps/reports/tools_and_job_state.mako index c2fcddc509b..558c3279925 100644 --- a/templates/webapps/reports/tools_and_job_state.mako +++ b/templates/webapps/reports/tools_and_job_state.mako @@ -6,14 +6,14 @@ %endif

-

Per-tool number of Jobs in function of the state

-

Listed in +

States of Jobs per Tool

+

Listed in %if descending == 1: descending %else: ascending %endif - order by + order by %if sorting == 0: Tool %elif sorting == 1: @@ -62,7 +62,7 @@ %endif

%if data[tool][0] + data[tool][1] != "--": - ${tool} + ${tool} %else: ${tool} %endif diff --git a/templates/webapps/reports/tools_and_job_state_peer_month.mako b/templates/webapps/reports/tools_and_job_state_per_month.mako similarity index 93% rename from templates/webapps/reports/tools_and_job_state_peer_month.mako rename to templates/webapps/reports/tools_and_job_state_per_month.mako index 924272d5350..f9b4458999f 100644 --- a/templates/webapps/reports/tools_and_job_state_peer_month.mako +++ b/templates/webapps/reports/tools_and_job_state_per_month.mako @@ -6,7 +6,7 @@ %endif
-

Number of Jobs Ok or in error peer month for ${tool}

+

States of Jobs for ${tool}

Listed in descending by month

%if data: From 295f3f93e1db89b13040b42c8945cd5491ccda02 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 26 Jul 2016 13:11:18 -0400 Subject: [PATCH 10/11] change css classes and html markup for new report pages --- templates/webapps/reports/history_and_dataset_per_user.mako | 4 +++- templates/webapps/reports/history_and_dataset_type.mako | 4 +++- templates/webapps/reports/history_per_user.mako | 4 +++- templates/webapps/reports/tool_execution_time.mako | 4 +++- templates/webapps/reports/tool_execution_time_per_month.mako | 4 +++- templates/webapps/reports/tools_and_job_state.mako | 4 +++- templates/webapps/reports/tools_and_job_state_per_month.mako | 4 +++- 7 files changed, 21 insertions(+), 7 deletions(-) diff --git a/templates/webapps/reports/history_and_dataset_per_user.mako b/templates/webapps/reports/history_and_dataset_per_user.mako index da36d43ad19..56de43b8971 100644 --- a/templates/webapps/reports/history_and_dataset_per_user.mako +++ b/templates/webapps/reports/history_and_dataset_per_user.mako @@ -5,7 +5,8 @@ ${render_msg( message, 'done' )} %endif -
+
+

Histories and Datasets per User

Listed in %if descending == 1: @@ -72,3 +73,4 @@ %endif

+ diff --git a/templates/webapps/reports/history_and_dataset_type.mako b/templates/webapps/reports/history_and_dataset_type.mako index 87e174b03c5..e0cf8e6e713 100644 --- a/templates/webapps/reports/history_and_dataset_type.mako +++ b/templates/webapps/reports/history_and_dataset_type.mako @@ -5,7 +5,8 @@ ${render_msg( message, 'done' )} %endif -
+
+

States of Datasets per History

Listed in %if descending == 1: @@ -71,3 +72,4 @@ %endif

+

diff --git a/templates/webapps/reports/history_per_user.mako b/templates/webapps/reports/history_per_user.mako index 76fa3443f3f..4c82e822f63 100644 --- a/templates/webapps/reports/history_per_user.mako +++ b/templates/webapps/reports/history_per_user.mako @@ -5,7 +5,8 @@ ${render_msg( message, 'done' )} %endif -
+
+

Histories per User

Listed in %if descending == 1: @@ -64,3 +65,4 @@ %endif

+
diff --git a/templates/webapps/reports/tool_execution_time.mako b/templates/webapps/reports/tool_execution_time.mako index 18376da8102..07e034d3159 100644 --- a/templates/webapps/reports/tool_execution_time.mako +++ b/templates/webapps/reports/tool_execution_time.mako @@ -2,7 +2,8 @@ <%namespace file="/message.mako" import="render_msg" /> -
+
+

Execution Time per Tool

Listed in %if descending == 1: @@ -75,3 +76,4 @@ %endif

+
diff --git a/templates/webapps/reports/tool_execution_time_per_month.mako b/templates/webapps/reports/tool_execution_time_per_month.mako index 636b826a083..0273f2e8991 100644 --- a/templates/webapps/reports/tool_execution_time_per_month.mako +++ b/templates/webapps/reports/tool_execution_time_per_month.mako @@ -2,7 +2,8 @@ <%namespace file="/message.mako" import="render_msg" /> -
+
+

Execution Time for ${tool} per Month

Listed in %if descending == 1: @@ -73,3 +74,4 @@ %endif

+
diff --git a/templates/webapps/reports/tools_and_job_state.mako b/templates/webapps/reports/tools_and_job_state.mako index 558c3279925..b093f621b58 100644 --- a/templates/webapps/reports/tools_and_job_state.mako +++ b/templates/webapps/reports/tools_and_job_state.mako @@ -5,7 +5,8 @@ ${render_msg( message, 'done' )} %endif -
+
+

States of Jobs per Tool

Listed in %if descending == 1: @@ -80,3 +81,4 @@ %endif

+
diff --git a/templates/webapps/reports/tools_and_job_state_per_month.mako b/templates/webapps/reports/tools_and_job_state_per_month.mako index f9b4458999f..5e358a99b78 100644 --- a/templates/webapps/reports/tools_and_job_state_per_month.mako +++ b/templates/webapps/reports/tools_and_job_state_per_month.mako @@ -5,7 +5,8 @@ ${render_msg( message, 'done' )} %endif -
+
+

States of Jobs for ${tool}

Listed in descending by month

@@ -42,3 +43,4 @@ %endif
+
From 8483f2ce8bffbdee0ee9ffd65b7212ed8c54e8a4 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 26 Jul 2016 17:07:37 -0400 Subject: [PATCH 11/11] Fix E731 --- lib/galaxy/webapps/reports/controllers/tools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/reports/controllers/tools.py b/lib/galaxy/webapps/reports/controllers/tools.py index cf11c968b41..56121836b89 100644 --- a/lib/galaxy/webapps/reports/controllers/tools.py +++ b/lib/galaxy/webapps/reports/controllers/tools.py @@ -195,7 +195,10 @@ class Tools( BaseUIController ): data = {} ordered_data = collections.OrderedDict() - field_sort = lambda first, second, field: descending if data[first][field] < data[second][field] else -descending + + def field_sort(first, second, field): + descending if data[first][field] < data[second][field] else -descending + sort_functions = [ lambda first, second: -descending if first.lower() < second.lower() else descending, lambda first, second: field_sort(first, second, "avg"),