From 4c1ce27e0a5ac61c93e2ffdc6f9205002a49bdfe Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 5 Jan 2022 14:03:44 -0500 Subject: [PATCH 1/2] Add job reports for node categories and execution times --- .../webapps/reports/controllers/jobs.py | 213 ++++++++++++++---- templates/page_base.mako | 9 +- templates/webapps/reports/index.mako | 2 + templates/webapps/reports/job_info.mako | 9 +- .../webapps/reports/jobs_per_month_all.mako | 5 +- ...obs_per_month_by_user_and_destination.mako | 115 ++++++++++ templates/webapps/reports/jobs_per_user.mako | 2 +- .../reports/jobs_per_user_by_destination.mako | 101 +++++++++ .../jobs_user_per_month_by_destination.mako | 84 +++++++ 9 files changed, 492 insertions(+), 48 deletions(-) create mode 100644 templates/webapps/reports/jobs_per_month_by_user_and_destination.mako create mode 100644 templates/webapps/reports/jobs_per_user_by_destination.mako create mode 100644 templates/webapps/reports/jobs_user_per_month_by_destination.mako diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index 8410af85798..630c1bfcce3 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -104,6 +104,17 @@ def get_spark_time(time_period): return time_period, _time_period +def get_curr_item(check_item, unique_items): + """ + When rendering by item and destination_id, + render the item uniquely. + """ + if check_item in unique_items: + return ('', unique_items) + unique_items.add(check_item) + return (check_item, unique_items) + + class SpecifiedDateListGrid(grids.Grid): class JobIdColumn(grids.IntegerColumn): @@ -154,6 +165,13 @@ class SpecifiedDateListGrid(grids.Grid): return query.filter(and_(model.Job.table.c.user_id == model.User.table.c.id, model.User.table.c.email == column_filter)) + class DestinationIdColumn(grids.GridColumn): + + def filter(self, trans, user, query, column_filter): + if column_filter == 'All': + return query + return query.filter(model.Job.table.c.destination_id == column_filter) + class SpecifiedDateColumn(grids.GridColumn): def filter(self, trans, user, query, column_filter): @@ -187,6 +205,9 @@ class SpecifiedDateListGrid(grids.Grid): StateColumn("State", key="state", attach_popup=False), + DestinationIdColumn("Destination Id", + key="destination_id", + attach_popup=False), ToolColumn("Tool Id", key="tool_id", link=(lambda item: dict(operation="tool_per_month", id=item.id, webapp="reports")), @@ -274,6 +295,9 @@ class Jobs(BaseUIController, ReportQueryBuilder): **kwd)) elif operation == "user_for_month": kwd['f-email'] = util.restore_text(kwd['email']) + elif operation == "user_for_month_by_destination": + kwd['f-email'] = util.restore_text(kwd['email']) + kwd['f-destination_id'] = kwd['destination_id'] elif operation == "user_per_month": # The received id is the job id, so we need to get the id of the user # that submitted the job. @@ -313,18 +337,38 @@ class Jobs(BaseUIController, ReportQueryBuilder): trends[key][job_day] += 1 return trends - def _calculate_job_table(self, sa_session, jobs_query): + def _calculate_job_table(self, sa_session, jobs_query, by_destination=False): jobs = [] + unique_month_year_strs = set() for row in sa_session.execute(jobs_query): month_name = row.date.strftime("%B") year = int(row.date.strftime("%Y")) - jobs.append(( - row.date.strftime("%Y-%m"), - row.total_jobs, - month_name, - year - )) + if str(by_destination).lower() == 'true': + month_year_str = "%s %s" % (month_name, year) + curr_month_year_str, unique_month_year_strs = get_curr_item(month_year_str, unique_month_year_strs) + if curr_month_year_str == '': + curr_month = '' + curr_year = '' + else: + curr_month =row.date.strftime("%B") + curr_year = row.date.strftime("%Y") + jobs.append(( + row.date.strftime("%Y-%m"), + row.total_jobs, + curr_month, + curr_year, + row.user_email, + row.destination_id, + row.execute_time + )) + else: + jobs.append(( + row.date.strftime("%Y-%m"), + row.total_jobs, + month_name, + year + )) return jobs @web.expose @@ -534,9 +578,11 @@ class Jobs(BaseUIController, ReportQueryBuilder): @web.expose def per_month_all(self, trans, **kwd): """ - Queries the DB for all jobs. Avoids monitor jobs. + Queries the DB for all jobs. Avoids monitor jobs. The + by_destination param will group by User.email and + Job.destination_id. """ - + by_destination = str(kwd.get('by_destination', False)).lower() message = '' PageSpec = namedtuple('PageSpec', ['entries', 'offset', 'page', 'pages_found']) @@ -570,26 +616,43 @@ class Jobs(BaseUIController, ReportQueryBuilder): monitor_user_id = get_monitor_id(trans, monitor_email) # Use to make the page table - jobs_by_month = sa.select((self.select_month(model.Job.table.c.create_time).label('date'), - sa.func.count(model.Job.table.c.id).label('total_jobs')), - whereclause=model.Job.table.c.user_id != monitor_user_id, - from_obj=[model.Job.table], - group_by=self.group_by_month(model.Job.table.c.create_time), - order_by=[_order], - offset=offset, - limit=limit) + if by_destination == 'true': + jobs_by_month = sa.select((self.select_month(model.Job.table.c.create_time).label('date'), + model.Job.table.c.destination_id.label('destination_id'), + sa.func.sum(model.Job.table.c.update_time - model.Job.table.c.create_time).label('execute_time'), + sa.func.count(model.Job.table.c.id).label('total_jobs'), + model.User.table.c.email.label('user_email')), + whereclause=model.Job.table.c.user_id != monitor_user_id, + from_obj=[sa.join(model.Job.table, model.User.table)], + group_by=['user_email', 'date', 'destination_id'], + order_by=[_order], + offset=offset, + limit=limit) + else: + jobs_by_month = sa.select((self.select_month(model.Job.table.c.create_time).label('date'), + sa.func.count(model.Job.table.c.id).label('total_jobs')), + whereclause=model.Job.table.c.user_id != monitor_user_id, + from_obj=[model.Job.table], + group_by=self.group_by_month(model.Job.table.c.create_time), + order_by=[_order], + offset=offset, + limit=limit) # Use to make sparkline all_jobs = sa.select((self.select_day(model.Job.table.c.create_time).label('date'), model.Job.table.c.id.label('id'))) trends = self._calculate_trends_for_jobs(trans.sa_session, all_jobs) - jobs = self._calculate_job_table(trans.sa_session, jobs_by_month) + jobs = self._calculate_job_table(trans.sa_session, jobs_by_month, by_destination=by_destination) pages_found = ceil(len(jobs) / float(entries)) page_specs = PageSpec(entries, offset, page, pages_found) - return trans.fill_template('/webapps/reports/jobs_per_month_all.mako', + if by_destination == 'true': + page = '/webapps/reports/jobs_per_month_by_user_and_destination.mako' + else: + page = '/webapps/reports/jobs_per_month_all.mako' + return trans.fill_template(page, order=order, arrow=arrow, sort_id=sort_id, @@ -674,6 +737,11 @@ class Jobs(BaseUIController, ReportQueryBuilder): @web.expose def per_user(self, trans, **kwd): + """ + Queries the DB for jobs per user. The by_destination + param will group by Job.destination_id. + """ + by_destination = str(kwd.get('by_destination', False)).lower() total_time = Timer() q_time = Timer() @@ -711,24 +779,42 @@ class Jobs(BaseUIController, ReportQueryBuilder): page = 1 jobs = [] - jobs_per_user = sa.select((model.User.table.c.email.label('user_email'), - sa.func.count(model.Job.table.c.id).label('total_jobs')), - from_obj=[sa.outerjoin(model.Job.table, model.User.table)], - group_by=['user_email'], - order_by=[_order], - offset=offset, - limit=limit) + if by_destination == 'true': + jobs_per_user = sa.select((model.User.table.c.email.label('user_email'), + sa.func.count(model.Job.table.c.id).label('total_jobs'), + model.Job.table.c.destination_id.label('destination_id')), + from_obj=[sa.outerjoin(model.Job.table, model.User.table)], + group_by=['user_email', 'destination_id'], + order_by=[_order], + offset=offset, + limit=limit) + + else: + jobs_per_user = sa.select((model.User.table.c.email.label('user_email'), + sa.func.count(model.Job.table.c.id).label('total_jobs')), + from_obj=[sa.outerjoin(model.Job.table, model.User.table)], + group_by=['user_email'], + order_by=[_order], + offset=offset, + limit=limit) q_time.start() + unique_users = set() for row in trans.sa_session.execute(jobs_per_user): if (row.user_email is None): - jobs.append(('Anonymous', - row.total_jobs)) + curr_user, unique_users = get_curr_item('Anonymous', unique_users) + if by_destination == 'true': + jobs.append((curr_user, row.destination_id, row.total_jobs)) + else: + jobs.append((curr_user, row.total_jobs)) elif (row.user_email == monitor_email): continue else: - jobs.append((row.user_email, - row.total_jobs)) + curr_user, unique_users = get_curr_item(row.user_email, unique_users) + if by_destination == 'true': + jobs.append((curr_user, row.destination_id, row.total_jobs)) + else: + jobs.append((curr_user, row.total_jobs)) q_time.stop() query1time = q_time.time_elapsed() @@ -773,7 +859,11 @@ class Jobs(BaseUIController, ReportQueryBuilder): total_time.stop() ttime = total_time.time_elapsed() - return trans.fill_template('/webapps/reports/jobs_per_user.mako', + if by_destination == 'true': + page = '/webapps/reports/jobs_per_user_by_destination.mako' + else: + page = '/webapps/reports/jobs_per_user.mako' + return trans.fill_template(page, order=order, arrow=arrow, sort_id=sort_id, @@ -789,6 +879,11 @@ class Jobs(BaseUIController, ReportQueryBuilder): @web.expose def user_per_month(self, trans, **kwd): + """ + Queries the DB for jobs per user per month. The + by_destination param will group by Job.destination_id. + """ + by_destination = str(kwd.get('by_destination', False)).lower() params = util.Params(kwd) message = '' @@ -799,12 +894,22 @@ class Jobs(BaseUIController, ReportQueryBuilder): arrow = specs.arrow _order = specs.exc_order - q = sa.select((self.select_month(model.Job.table.c.create_time).label('date'), - sa.func.count(model.Job.table.c.id).label('total_jobs')), - whereclause=model.User.table.c.email == email, - from_obj=[sa.join(model.Job.table, model.User.table)], - group_by=self.group_by_month(model.Job.table.c.create_time), - order_by=[_order]) + if by_destination == 'true': + q = sa.select((self.select_month(model.Job.table.c.create_time).label('date'), + model.Job.table.c.destination_id.label('destination_id'), + sa.func.sum(model.Job.table.c.update_time - model.Job.table.c.create_time).label('execute_time'), + sa.func.count(model.Job.table.c.id).label('total_jobs')), + whereclause=model.User.table.c.email == email, + from_obj=[sa.join(model.Job.table, model.User.table)], + group_by=['date', 'destination_id'], + order_by=[_order]) + else: + q = sa.select((self.select_month(model.Job.table.c.create_time).label('date'), + sa.func.count(model.Job.table.c.id).label('total_jobs')), + whereclause=model.User.table.c.email == email, + from_obj=[sa.join(model.Job.table, model.User.table)], + group_by=self.group_by_month(model.Job.table.c.create_time), + order_by=[_order]) all_jobs_per_user = sa.select((model.Job.table.c.create_time.label('date'), model.Job.table.c.id.label('job_id')), @@ -828,19 +933,41 @@ class Jobs(BaseUIController, ReportQueryBuilder): trends[key][job_day] += 1 jobs = [] + unique_month_year_strs = set() for row in trans.sa_session.execute(q): - jobs.append((row.date.strftime("%Y-%m"), - row.total_jobs, - row.date.strftime("%B"), - row.date.strftime("%Y"))) - return trans.fill_template('/webapps/reports/jobs_user_per_month.mako', + if by_destination == 'true': + month_year_str = "%s %s" % (row.date.strftime("%B"), row.date.strftime("%Y")) + curr_month_year_str, unique_month_year_strs = get_curr_item(month_year_str, unique_month_year_strs) + if curr_month_year_str == '': + curr_month = '' + curr_year = '' + else: + curr_month =row.date.strftime("%B") + curr_year = row.date.strftime("%Y") + jobs.append((row.date.strftime("%Y-%m"), + row.execute_time, + row.total_jobs, + curr_month, + curr_year, + row.destination_id)) + else: + jobs.append((row.date.strftime("%Y-%m"), + row.total_jobs, + row.date.strftime("%B"), + row.date.strftime("%Y"))) + if by_destination == 'true': + page = '/webapps/reports/jobs_user_per_month_by_destination.mako' + else: + page = '/webapps/reports/jobs_user_per_month.mako' + return trans.fill_template(page, order=order, arrow=arrow, sort_id=sort_id, id=kwd.get('id'), trends=trends, email=util.sanitize_text(email), - jobs=jobs, message=message) + jobs=jobs, + message=message) @web.expose def per_tool(self, trans, **kwd): diff --git a/templates/page_base.mako b/templates/page_base.mako index ca465480731..783dc3dd253 100644 --- a/templates/page_base.mako +++ b/templates/page_base.mako @@ -3,7 +3,13 @@ <%def name="get_page_url( sort_id, order, *args, **kwargs )"> + %try: + %if str(by_destination).lower() == "true": + ${kwargs.get("page")} + %endif + %except NameError: ${kwargs.get("page")} + %endtry <%! @@ -44,11 +50,12 @@ -<%def name="get_entry_selector(controller, action, entries, sort_id, order)"> +<%def name="get_entry_selector(controller, action, entries, sort_id, order, by_destination=False)">
+ %try: %if spark_limit: diff --git a/templates/webapps/reports/index.mako b/templates/webapps/reports/index.mako index 70e983edac3..0d874534aba 100644 --- a/templates/webapps/reports/index.mako +++ b/templates/webapps/reports/index.mako @@ -49,8 +49,10 @@ + +
diff --git a/templates/webapps/reports/job_info.mako b/templates/webapps/reports/job_info.mako index 9b844e9d90e..8f246bb5039 100644 --- a/templates/webapps/reports/job_info.mako +++ b/templates/webapps/reports/job_info.mako @@ -47,16 +47,20 @@ ${job.job_runner_external_id} - Remote Host + Remote Host + Destination Id + Destination params - + %if job.galaxy_session and job.galaxy_session.remote_host: ${job.galaxy_session.remote_host} %else: no remote host %endif + ${job.destination_id} + ${job.destination_params} Command Line @@ -93,3 +97,4 @@ + diff --git a/templates/webapps/reports/jobs_per_month_all.mako b/templates/webapps/reports/jobs_per_month_all.mako index 33165df257d..2bc254c88bb 100644 --- a/templates/webapps/reports/jobs_per_month_all.mako +++ b/templates/webapps/reports/jobs_per_month_all.mako @@ -82,7 +82,10 @@ ${get_css()} ${job[1]} - ${make_sparkline(key, trends[key], "bar", "/ day")} + %try: + ${make_sparkline(key, trends[key], "bar", "/ day")} + %except KeyError: + %endtry <% diff --git a/templates/webapps/reports/jobs_per_month_by_user_and_destination.mako b/templates/webapps/reports/jobs_per_month_by_user_and_destination.mako new file mode 100644 index 00000000000..b7316b06362 --- /dev/null +++ b/templates/webapps/reports/jobs_per_month_by_user_and_destination.mako @@ -0,0 +1,115 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/spark_base.mako" import="make_sparkline" /> +<%namespace file="/sorting_base.mako" import="get_sort_url, get_css" /> +<%namespace file="/page_base.mako" import="get_pages, get_entry_selector" /> + +<% + import datetime +%> + +%if message: + ${render_msg( message, 'done' )} +%endif + +<% + page = page_specs.page + offset = page_specs.offset + entries = page_specs.entries +%> + +${get_css()} + + +
+
+ + + + + + +
+ ${get_pages(sort_id, order, page_specs, 'jobs', 'per_month_all')} + +

Jobs Per Month by User/Node type

+
+ Click Month to view details. + Graph goes from the 1st to the last of the month. +
+
+ ${get_entry_selector("jobs", "per_month_all", page_specs.entries, sort_id, order, by_destination=True)} +
+ + + %if len( jobs ) == 0: + + %else: + + + + + + + + + + <% + ctr = 0 + entries = 1 + %> + %for job in jobs: + <% key = str(job[2]) + str(job[3]) %> + + %if entries > page_specs.entries: + <%break%> + %endif + + %if ctr % 2 == 1: + + %else: + + %endif + + + + + + + + %try: + ${make_sparkline(key, trends[key], "bar", "/ day")} + %except KeyError: + %endtry + + + <% + ctr += 1 + entries += 1 + %> + %endfor + %endif +
There are no jobs.
+ ${get_sort_url(sort_id, order, 'date', 'jobs', 'per_month_all', 'Month', page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} + + ${get_sort_url(sort_id, order, 'date', 'jobs', 'per_month_all', 'User', page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} + + ${get_sort_url(sort_id, order, 'date', 'jobs', 'per_month_all', 'Node Type', page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} + + ${get_sort_url(sort_id, order, 'total_jobs', 'jobs', 'per_month_all', 'Jobs', page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} + + Total Execution Time: seconds + + Total Execution Time: hh:mm:ss +
+ + ${job[2]} ${job[3]} + + ${job[4]}${job[5]}${job[1]}${job[6].seconds}${datetime.timedelta(seconds=job[6].seconds)}
+
+
+ diff --git a/templates/webapps/reports/jobs_per_user.mako b/templates/webapps/reports/jobs_per_user.mako index c928f797397..85a52b7671e 100644 --- a/templates/webapps/reports/jobs_per_user.mako +++ b/templates/webapps/reports/jobs_per_user.mako @@ -20,7 +20,6 @@ ${get_css()} %> -${q1time}, ${q2time}, ${ttime}
@@ -95,3 +94,4 @@ ${q1time}, ${q2time}, ${ttime} + diff --git a/templates/webapps/reports/jobs_per_user_by_destination.mako b/templates/webapps/reports/jobs_per_user_by_destination.mako new file mode 100644 index 00000000000..02aab912345 --- /dev/null +++ b/templates/webapps/reports/jobs_per_user_by_destination.mako @@ -0,0 +1,101 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/spark_base.mako" import="make_sparkline, make_spark_settings" /> +<%namespace file="/sorting_base.mako" import="get_sort_url, get_css" /> +<%namespace file="/page_base.mako" import="get_pages, get_entry_selector" /> + +<%! + import re +%> + +%if message: + ${render_msg( message, 'done' )} +%endif + +${get_css()} +<% + page = page_specs.page + offset = page_specs.offset + entries = page_specs.entries +%> + + +
+
+
+ + + + + +
+ ${get_pages(sort_id, order, page_specs, 'jobs', 'per_user', by_destination=True, spark_time=time_period)} + +

Jobs Per User / Node Type

+
+ Click User to view details. + Graph goes from present to past + ${make_spark_settings("jobs", "per_user", spark_limit, sort_id, order, time_period, page=page, offset=offset, entries=entries, by_destination=True)} +
+
+ ${get_entry_selector("jobs", "per_user", page_specs.entries, sort_id, order, by_destination=True)} +
+ + %if len( jobs ) == 0: + + %else: + + + + + + + <% + ctr = 0 + entries = 1 + %> + %for job in jobs: + <% key = re.sub(r'\W+', '', job[0]) %> + + %if entries > page_specs.entries: + <%break%> + %endif + + %if ctr % 2 == 1: + + %else: + + %endif + + + + + %try: + ${make_sparkline(key, trends[key], "bar", "/ " + time_period[:-1])} + %except KeyError: + %endtry + + + <% + ctr += 1 + entries += 1 + %> + %endfor + %endif +
There are no jobs.
+ ${get_sort_url(sort_id, order, 'user_email', 'jobs', 'per_user', 'User', spark_time=time_period, page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} + + ${get_sort_url(sort_id, order, 'destination_id', 'jobs', 'per_user', 'Node Type', spark_time=time_period, page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} + + ${get_sort_url(sort_id, order, 'total_jobs', 'jobs', 'per_user', 'Total Jobs', spark_time=time_period, page=page, offset=offset, entries=entries, by_destination=True)} + ${arrow} +
+ + ${job[0]} + + ${job[1]}${job[2]}
+
+
+ diff --git a/templates/webapps/reports/jobs_user_per_month_by_destination.mako b/templates/webapps/reports/jobs_user_per_month_by_destination.mako new file mode 100644 index 00000000000..e8a3877e8a7 --- /dev/null +++ b/templates/webapps/reports/jobs_user_per_month_by_destination.mako @@ -0,0 +1,84 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/spark_base.mako" import="make_sparkline" /> +<%namespace file="/sorting_base.mako" import="get_sort_url, get_css" /> + +<% + import datetime + from galaxy import util +%> + +%if message: + ${render_msg( message, 'done' )} +%endif + +${get_css()} + +<% + _email = util.restore_text( email ) +%> + + +
+
+

Jobs per month for user "${_email}" / node type

+

+

Click Total Jobs to see the user's jobs for that month

+

Graph goes from first of the month to the last

+

+ + %if len( jobs ) == 0: + + + + %else: + + + + + + + + <% ctr = 0 %> + %for job in jobs: + <% key = job[3] + job[4] %> + %if ctr % 2 == 1: + + %else: + + %endif + + + + + + %try: + ${make_sparkline(key, trends[key], "bar", "/ day")} + %except KeyError: + %endtry + + <% ctr += 1 %> + %endfor + %endif +
+ There are no jobs for user "${ _email }" +
+ ${get_sort_url(sort_id, order, 'date', 'jobs', 'user_per_month', 'Month', email=email, by_destination=True)} + ${arrow} + + Node Type + + ${get_sort_url( sort_id, order, 'total_jobs', 'jobs', 'user_per_month', 'Total Jobs', email=email, by_destination=True)} + ${arrow} + + Total Execution Time: seconds + + Total Execution Time: hh:mm:ss +
${job[3]} ${job[4]}${job[5]} + + ${job[2]} + + ${job[1].seconds}${datetime.timedelta(seconds=job[1].seconds)}
+
+
+ From badeb09a8ab4160844001a95d8a217704d9e7b8d Mon Sep 17 00:00:00 2001 From: greg Date: Wed, 5 Jan 2022 14:13:47 -0500 Subject: [PATCH 2/2] flake8 fixes --- lib/galaxy/webapps/reports/controllers/jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/jobs.py b/lib/galaxy/webapps/reports/controllers/jobs.py index 630c1bfcce3..8b9c2b755c5 100644 --- a/lib/galaxy/webapps/reports/controllers/jobs.py +++ b/lib/galaxy/webapps/reports/controllers/jobs.py @@ -351,7 +351,7 @@ class Jobs(BaseUIController, ReportQueryBuilder): curr_month = '' curr_year = '' else: - curr_month =row.date.strftime("%B") + curr_month = row.date.strftime("%B") curr_year = row.date.strftime("%Y") jobs.append(( row.date.strftime("%Y-%m"), @@ -942,7 +942,7 @@ class Jobs(BaseUIController, ReportQueryBuilder): curr_month = '' curr_year = '' else: - curr_month =row.date.strftime("%B") + curr_month = row.date.strftime("%B") curr_year = row.date.strftime("%Y") jobs.append((row.date.strftime("%Y-%m"), row.execute_time,