From 1ba753ca54b5a20b03f28514894b71c2c966780a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 18 Apr 2019 12:53:34 -0400 Subject: [PATCH 001/213] Catch and filter *all* Slurm cgroup-related messages --- lib/galaxy/jobs/runners/slurm.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 333fd55d563..ec2ce131d45 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -28,13 +28,9 @@ SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.', ': Exceeded step memory limit at some point.'] SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB -SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE = re.compile(r"""slurmstepd: error: task/cgroup: unable to add task\[pid=\d+\] to memory cg '\(null\)'$""") -SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: xcgroup_instantiate: unable to create cgroup '[^']+' : No space left on device$""") -SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: jobacct_gather/cgroup: unable to instanciate (job|user) \d+ memory cgroup$""") +SLURM_CGROUP_RE = re.compile(r"""slurmstepd: .*cgroup.*$""") SLURM_TOP_WARNING_RES = ( - SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE, - SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE, - SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE + SLURM_CGROUP_RE, ) # These messages are returned to the user From 29dfc3085be63664e6e903dbdb2ef44ccd3ce234 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 20 Apr 2019 08:18:39 -0400 Subject: [PATCH 002/213] Track basename at the time of creation for datasets. --- lib/galaxy/jobs/__init__.py | 2 +- lib/galaxy/managers/hdas.py | 6 ++-- lib/galaxy/model/__init__.py | 14 ++++++++- lib/galaxy/model/mapping.py | 1 + .../versions/0152_add_metadata_file_uuid.py | 21 ++++---------- .../versions/0154_created_from_basename.py | 29 +++++++++++++++++++ lib/galaxy/model/store/discover.py | 16 ++++++++++ lib/galaxy/tools/actions/__init__.py | 5 ++++ lib/galaxy/tools/data_fetch.py | 3 ++ lib/galaxy/tools/verify/interactor.py | 2 +- test/api/test_libraries.py | 3 +- test/functional/tools/output_format.xml | 2 ++ .../tools/tool_provided_metadata_9.xml | 4 ++- 13 files changed, 85 insertions(+), 23 deletions(-) create mode 100644 lib/galaxy/model/migrate/versions/0154_created_from_basename.py diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index e3460c38fc8..8cbc4bff221 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -46,7 +46,7 @@ log = logging.getLogger(__name__) # that import Galaxy internals - but it shouldn't be used in Galaxy's code # itself. TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' -TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey'] +TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename'] # Override with config.default_job_shell. DEFAULT_JOB_SHELL = '/bin/bash' diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index 1842665ea78..08bd08e505f 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -300,7 +300,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize 'annotation', - 'api_type' + 'api_type', + 'created_from_basename', ], include_keys_from='summary') self.add_view('extended', [ @@ -354,7 +355,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize # TODO: to DatasetAssociationSerializer 'accessible' : lambda i, k, user=None, **c: self.manager.is_accessible(i, user, **c), 'api_type' : lambda *a, **c: 'file', - 'type' : lambda *a, **c: 'file' + 'type' : lambda *a, **c: 'file', + 'created_from_basename' : lambda i, k, **c: i.created_from_basename, }) def serialize(self, hda, keys, user=None, **context): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fe26813a4e2..00684f71ae2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2510,6 +2510,16 @@ class DatasetInstance(object): """Detects whether there is any data""" return self.dataset.has_data() + def get_created_from_basename(self): + return self.dataset.created_from_basename + + def set_created_from_basename(self, created_from_basename): + if self.dataset.created_from_basename is not None: + raise Exception("Underlying dataset already has a created_from_basename set.") + self.dataset.created_from_basename = created_from_basename + + created_from_basename = property(get_created_from_basename, set_created_from_basename) + def get_raw_data(self): """Returns the full data. To stream it open the file_name and read/write as needed""" return self.datatype.get_raw_data(self) @@ -3393,6 +3403,7 @@ class LibraryDataset(RepresentById): state=ldda.state, name=ldda.name, file_name=ldda.file_name, + created_from_basename=ldda.created_from_basename, uploaded_by=ldda.user.email, message=ldda.message, date_uploaded=ldda.create_time.isoformat(), @@ -3545,7 +3556,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__, genome_build=ldda.dbkey, misc_info=ldda.info, - misc_blurb=ldda.blurb) + misc_blurb=ldda.blurb, + created_from_basename=ldda.created_from_basename) if ldda.dataset.uuid is None: rval['uuid'] = None else: diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f7f563c54c8..25f91256140 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -263,6 +263,7 @@ model.Dataset.table = Table( Column("object_store_id", TrimmedString(255), index=True), Column("external_filename", TEXT), Column("_extra_files_path", TEXT), + Column("created_from_basename", TEXT), Column('file_size', Numeric(15, 0)), Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) diff --git a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py index f9d26724a37..0f25287393c 100644 --- a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py +++ b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py @@ -6,9 +6,10 @@ from __future__ import print_function import logging -from sqlalchemy import Column, MetaData, Table +from sqlalchemy import Column, MetaData from galaxy.model.custom_types import UUIDType +from galaxy.model.migrate.versions.util import add_column, drop_column log = logging.getLogger(__name__) @@ -19,14 +20,8 @@ def upgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - metadata_file_table = Table("metadata_file", metadata, autoload=True) - - try: - uuid_column = Column('uuid', UUIDType()) - uuid_column.create(metadata_file_table) - assert uuid_column is metadata_file_table.c.uuid - except Exception: - log.exception("Adding column 'uuid' to `MetadataFile` table failed.") + uuid_column = Column('uuid', UUIDType()) + add_column(uuid_column, 'metadata_file', metadata) def downgrade(migrate_engine): @@ -34,10 +29,4 @@ def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - metadata_file_table = Table("metadata_file", metadata, autoload=True) - - try: - column = metadata_file_table.c.uuid - column.drop() - except Exception: - log.exception("Dropping 'uuid' column from `metadata_file` table failed.") + drop_column('uuid', 'metadata_file', metadata) diff --git a/lib/galaxy/model/migrate/versions/0154_created_from_basename.py b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py new file mode 100644 index 00000000000..c4664b48e97 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py @@ -0,0 +1,29 @@ +""" +Adds created_from_basename to dataset. +""" +import datetime +import logging + +from sqlalchemy import Column, MetaData, TEXT + +from galaxy.model.migrate.versions.util import add_column, drop_column + +now = datetime.datetime.utcnow +log = logging.getLogger(__name__) +metadata = MetaData() + + +def upgrade(migrate_engine): + metadata.bind = migrate_engine + print(__doc__) + metadata.reflect() + + created_from_basename_column = Column("created_from_basename", TEXT, default=None) + add_column(created_from_basename_column, 'dataset', metadata) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + + drop_column('created_from_basename', 'dataset', metadata) diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 697aefc15b0..2f5964d4ed9 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -55,6 +55,7 @@ class ModelPersistenceContext(object): tag_list=[], sources=[], hashes=[], + created_from_basename=None, ): sa_session = self.sa_session @@ -116,6 +117,9 @@ class ModelPersistenceContext(object): hash_object.hash_value = hash_dict["hash_value"] primary_data.dataset.hashes.append(hash_object) + if created_from_basename is not None: + primary_data.created_from_basename = created_from_basename + self.flush() if tag_list: @@ -204,6 +208,7 @@ class ModelPersistenceContext(object): sources = discovered_file.match.sources hashes = discovered_file.match.hashes + created_from_basename = discovered_file.match.created_from_basename dataset = self.create_dataset( ext=ext, @@ -217,6 +222,7 @@ class ModelPersistenceContext(object): tag_list=tag_list, sources=sources, hashes=hashes, + created_from_basename=created_from_basename, ) log.debug( "(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s", @@ -475,6 +481,8 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold sources = fields_match.sources hashes = fields_match.hashes + created_from_basename = fields_match.created_from_basename + model_persistence_context.create_dataset( ext=ext, designation=designation, @@ -487,6 +495,7 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold link_data=link_data, sources=sources, hashes=hashes, + created_from_basename=created_from_basename, ) @@ -517,6 +526,8 @@ def persist_hdas(elements, model_persistence_context): sources = fields_match.sources hashes = fields_match.hashes + created_from_basename = fields_match.created_from_basename + dataset = model_persistence_context.create_dataset( ext=ext, designation=designation, @@ -529,6 +540,7 @@ def persist_hdas(elements, model_persistence_context): primary_data=primary_dataset, sources=sources, hashes=hashes, + created_from_basename=created_from_basename, ) dataset.raw_set_dataset_state('ok') if not hda_id: @@ -701,6 +713,10 @@ class JsonCollectedDatasetMatch(object): def hashes(self): return self.as_dict.get("hashes", []) + @property + def created_from_basename(self): + return self.as_dict.get("created_from_basename") + class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch): diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 7f4ecd03c92..676e2cd5a4a 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -1,5 +1,6 @@ import json import logging +import os import re from json import dumps @@ -392,6 +393,10 @@ class DefaultToolAction(object): break data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False) + if create_datasets: + from_work_dir = output.from_work_dir + if from_work_dir is not None: + data.dataset.created_from_basename = os.path.basename(from_work_dir) if hidden is None: hidden = output.hidden if not hidden and dataset_collection_elements is not None: # Mapping over a collection - hide datasets diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index 75eaa204d48..90b35f64c69 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -108,6 +108,7 @@ def _fetch_target(upload_config, target): dbkey = item.get("dbkey", "?") requested_ext = item.get("ext", "auto") info = item.get("info", None) + created_from_basename = item.get("created_from_basename", None) tags = item.get("tags", []) object_id = item.get("object_id", None) link_data_only = upload_config.link_data_only @@ -162,6 +163,8 @@ def _fetch_target(upload_config, target): rval["object_id"] = object_id if tags: rval["tags"] = tags + if created_from_basename: + rval["created_from_basename"] = created_from_basename return rval elements = elements_tree_map(_resolve_src, items) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 81435bcf2d9..89c7d066efa 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -194,7 +194,7 @@ class GalaxyInteractorApi(object): """ metadata = attributes.get('metadata', {}).copy() for key, value in metadata.copy().items(): - if key not in ['name', 'info', 'tags']: + if key not in ['name', 'info', 'tags', 'created_from_basename']: new_key = "metadata_%s" % key metadata[new_key] = metadata[key] del metadata[key] diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index 8535511e480..a9f8d5f4f05 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -101,7 +101,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): def test_fetch_upload_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder("flat_zip") - items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed"}] + items = [{"src": "files", "dbkey": "hg19", "info": "my cool bed", "created_from_basename": "4.bed"}] targets = [{ "destination": destination, "items": items @@ -117,6 +117,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): assert dataset["genome_build"] == "hg19", dataset assert dataset["misc_info"] == "my cool bed", dataset assert dataset["file_ext"] == "bed", dataset + assert dataset["created_from_basename"] == "4.bed" def test_fetch_zip_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder("flat_zip") diff --git a/test/functional/tools/output_format.xml b/test/functional/tools/output_format.xml index 4f95a20f238..8c5005879ef 100644 --- a/test/functional/tools/output_format.xml +++ b/test/functional/tools/output_format.xml @@ -32,11 +32,13 @@ + + diff --git a/test/functional/tools/tool_provided_metadata_9.xml b/test/functional/tools/tool_provided_metadata_9.xml index 797ede1c871..285d5c88089 100644 --- a/test/functional/tools/tool_provided_metadata_9.xml +++ b/test/functional/tools/tool_provided_metadata_9.xml @@ -10,7 +10,8 @@ "name": "my dynamic name", "ext": "txt", "info": "my dynamic info", - "dbkey": "cust1" + "dbkey": "cust1", + "created_from_basename": "my name.txt" }} @@ -29,6 +30,7 @@ + From cd37a846a27527aae800c13a59ffb8eec4179df9 Mon Sep 17 00:00:00 2001 From: Sebastian Kapust Date: Sat, 20 Apr 2019 17:49:01 +0200 Subject: [PATCH 003/213] Vue rewrite of admin jobs page --- .../galaxy/scripts/components/admin/Jobs.vue | 371 ++++++++++++++++++ .../galaxy/scripts/entry/admin/AdminRouter.js | 6 + .../scripts/entry/panels/admin-panel.js | 1 + lib/galaxy/webapps/galaxy/buildapp.py | 1 + .../webapps/galaxy/controllers/admin.py | 104 ++--- templates/admin/jobs.mako | 209 ---------- 6 files changed, 437 insertions(+), 255 deletions(-) create mode 100644 client/galaxy/scripts/components/admin/Jobs.vue delete mode 100644 templates/admin/jobs.mako diff --git a/client/galaxy/scripts/components/admin/Jobs.vue b/client/galaxy/scripts/components/admin/Jobs.vue new file mode 100644 index 00000000000..5efa1cec465 --- /dev/null +++ b/client/galaxy/scripts/components/admin/Jobs.vue @@ -0,0 +1,371 @@ + + + + + diff --git a/client/galaxy/scripts/entry/admin/AdminRouter.js b/client/galaxy/scripts/entry/admin/AdminRouter.js index 1eeae6a2138..8e8aa024e06 100644 --- a/client/galaxy/scripts/entry/admin/AdminRouter.js +++ b/client/galaxy/scripts/entry/admin/AdminRouter.js @@ -7,6 +7,7 @@ import QueryStringParsing from "utils/query-string-parsing"; import Router from "layout/router"; import DataTables from "components/admin/DataTables.vue"; import DataTypes from "components/admin/DataTypes.vue"; +import Jobs from "components/admin/Jobs.vue"; import DataManagerView from "components/admin/DataManager/DataManagerView.vue"; import DataManagerRouter from "components/admin/DataManager/DataManagerRouter.vue"; import Register from "components/login/Register.vue"; @@ -33,6 +34,7 @@ export const getAdminRouter = (Galaxy, options) => { "(/)admin(/)form(/)(:form_id)": "show_form", "(/)admin/data_tables": "show_data_tables", "(/)admin/data_types": "show_data_types", + "(/)admin/jobs": "show_jobs", "(/)admin/data_manager*path": "show_data_manager", "*notFound": "not_found" }, @@ -115,6 +117,10 @@ export const getAdminRouter = (Galaxy, options) => { this._display_vue_helper(DataTypes); }, + show_jobs: function() { + this._display_vue_helper(Jobs); + }, + show_error_stack: function() { this._display_vue_helper(ErrorStack); }, diff --git a/client/galaxy/scripts/entry/panels/admin-panel.js b/client/galaxy/scripts/entry/panels/admin-panel.js index a94035e382e..44e60acc517 100644 --- a/client/galaxy/scripts/entry/panels/admin-panel.js +++ b/client/galaxy/scripts/entry/panels/admin-panel.js @@ -39,6 +39,7 @@ var AdminPanel = Backbone.View.extend({ { title: _l("Manage jobs"), url: "admin/jobs", + target: "__use_router__", id: "admin-link-jobs" }, { diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 6eb8fa7a76c..9289ab0b1f5 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -101,6 +101,7 @@ def app_factory(global_conf, load_app_kwds={}, **kwargs): webapp.add_client_route('/admin/data_tables', 'admin') webapp.add_client_route('/admin/data_types', 'admin') + webapp.add_client_route('/admin/jobs', 'admin') webapp.add_client_route('/admin/data_manager{path_info:.*}', 'admin') webapp.add_client_route('/admin/error_stack', 'admin') webapp.add_client_route('/admin/users', 'admin') diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index f001821285a..63d0f2039a1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -1600,14 +1600,25 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP return {'message' : 'User \'%s\' has been updated with %d associated roles and %d associated groups (private roles are not displayed).' % (user.email, len(in_roles) - 1, len(in_groups))} @web.expose + @web.json @web.require_admin - def jobs(self, trans, stop=[], stop_msg=None, cutoff=180, job_lock=None, ajl_submit=None, **kwd): + def jobs_control(self, trans, job_lock=None, **kwd): + if job_lock is not None: + job_lock = True if job_lock == 'true' else False + galaxy.queue_worker.send_control_task(trans.app, 'admin_job_lock', kwargs={'job_lock': job_lock}) + job_lock = trans.app.job_manager.job_lock + return {'job_lock': job_lock} + + @web.expose + @web.json + @web.require_admin + def jobs_list(self, trans, stop=[], stop_msg=None, cutoff=180, **kwd): deleted = [] - msg = None - status = None + message = kwd.get('message', '') + status = kwd.get('status', 'info') job_ids = util.listify(stop) if job_ids and stop_msg in [None, '']: - msg = 'Please enter an error message to display to the user describing why the job was terminated' + message = 'Please enter an error message to display to the user describing why the job was terminated' status = 'error' elif job_ids: if stop_msg[-1] not in PUNCTUATION: @@ -1624,24 +1635,14 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP trans.app.job_manager.stop(job, message=error_msg) deleted.append(str(job_id)) if deleted: - msg = 'Queued job' + message = 'Queued job' if len(deleted) > 1: - msg += 's' - msg += ' for deletion: ' - msg += ', '.join(deleted) + message += 's' + message += ' for deletion: ' + message += ', '.join(deleted) status = 'done' trans.sa_session.flush() - if ajl_submit: - if job_lock == 'on': - galaxy.queue_worker.send_control_task(trans.app, 'admin_job_lock', - kwargs={'job_lock': True}) - job_lock = True - else: - galaxy.queue_worker.send_control_task(trans.app, 'admin_job_lock', - kwargs={'job_lock': False}) - job_lock = False - else: - job_lock = trans.app.job_manager.job_lock + job_lock = trans.app.job_manager.job_lock cutoff_time = datetime.utcnow() - timedelta(seconds=int(cutoff)) jobs = trans.sa_session.query(trans.app.model.Job) \ .filter(and_(trans.app.model.Job.table.c.update_time < cutoff_time, @@ -1655,33 +1656,44 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP or_(trans.app.model.Job.state == trans.app.model.Job.states.ERROR, trans.app.model.Job.state == trans.app.model.Job.states.OK))) \ .order_by(trans.app.model.Job.table.c.update_time.desc()).all() - last_updated = {} - for job in jobs: - delta = datetime.utcnow() - job.update_time - if delta.days > 0: - last_updated[job.id] = '%s hours' % (delta.days * 24 + int(delta.seconds / 60 / 60)) - elif delta > timedelta(minutes=59): - last_updated[job.id] = '%s hours' % int(delta.seconds / 60 / 60) - else: - last_updated[job.id] = '%s minutes' % int(delta.seconds / 60) - finished = {} - for job in recent_jobs: - delta = datetime.utcnow() - job.update_time - if delta.days > 0: - finished[job.id] = '%s hours' % (delta.days * 24 + int(delta.seconds / 60 / 60)) - elif delta > timedelta(minutes=59): - finished[job.id] = '%s hours' % int(delta.seconds / 60 / 60) - else: - finished[job.id] = '%s minutes' % int(delta.seconds / 60) - return trans.fill_template('/admin/jobs.mako', - jobs=jobs, - recent_jobs=recent_jobs, - last_updated=last_updated, - finished=finished, - cutoff=cutoff, - msg=msg, - status=status, - job_lock=job_lock) + + def prepare_jobs_list(jobs): + res = [] + for job in jobs: + delta = datetime.utcnow() - job.update_time + update_time = "" + if delta.days > 0: + update_time = '%s hours ago' % (delta.days * 24 + int(delta.seconds / 60 / 60)) + elif delta > timedelta(minutes=59): + update_time = '%s hours ago' % int(delta.seconds / 60 / 60) + else: + update_time = '%s minutes ago' % int(delta.seconds / 60) + inputs = "" + try: + inputs = ", ".join(['{} {}'.format(da.dataset.id, da.dataset.state) for da in job.input_datasets]) + except Exception: + inputs = 'Unable to determine inputs' + res.append({ + 'job_info': { + 'id': job.id, + 'info_url': "{}?jobid={}".format(web.url_for(controller="admin", action="job_info"), job.id) + }, + 'user': job.history.user.email if job.history and job.history.user else 'anonymous', + 'update_time': update_time, + 'tool_id': job.tool_id, + 'state': job.state, + 'input_dataset': inputs, + 'command_line': job.command_line, + 'job_runner_name': job.job_runner_name, + 'job_runner_external_id': job.job_runner_external_id + }) + return res + return {'jobs': prepare_jobs_list(jobs), + 'recent_jobs': prepare_jobs_list(recent_jobs), + 'cutoff': cutoff, + 'message': message, + 'status': status, + 'job_lock': job_lock} @web.expose @web.require_admin diff --git a/templates/admin/jobs.mako b/templates/admin/jobs.mako deleted file mode 100644 index 36a372044f9..00000000000 --- a/templates/admin/jobs.mako +++ /dev/null @@ -1,209 +0,0 @@ -<%inherit file="/base.mako"/> -<%namespace file="/message.mako" import="render_msg" /> - -<%def name="javascripts()"> - ${parent.javascripts()} - - - -<%def name="title()">Jobs - -

Jobs

- -%if message: - ${render_msg( message, status )} -%endif - -

- Unfinished and recently finished jobs are displayed on this page. The - 'cutoff' input box will do two things -- it will limit the display of - unfinished jobs to only those jobs that have not had their job state - updated recently, and it will limit the recently finished jobs list to only - displaying jobs that have finished since the cutoff. -

-

- If any jobs are displayed, you may choose to stop them. Your stop message - will be displayed to the user as: "This job was stopped by an - administrator: <YOUR MESSAGE> For more information or help, - report this error". -

- -%if jobs: -
-

- Unfinished Jobs: These jobs are unfinished and have had their state updated in the previous ${cutoff} seconds. -

- - - - - - - - - - - - - - %for job in jobs: - - - %if job.history and job.history.user: - - %else: - - %endif - - - - <% - try: - inputs = ", ".join( [ '%s %s' % ( da.dataset.id, da.dataset.state ) for da in job.input_datasets ] ) - except: - inputs = 'Unable to determine inputs' - %> - - - - - - %endfor -
Job IDUserLast UpdateToolStateInputsCommand LineJob RunnerPID/Cluster ID
- - ${job.id}${job.history.user.email|h}anonymous${last_updated[job.id]} ago${job.tool_id|h}${job.state}${inputs}${job.command_line|h}${job.job_runner_name|h}${job.job_runner_external_id}
-

-

-
- Stop Jobs -
-
-
- -
- -
-
- to be displayed to the user -
-
-
- -
-
-
-

-

-%else: -
There are no unfinished jobs to show with current cutoff time.
-

-%endif - -%if recent_jobs: -

- Recent Jobs: These jobs have completed in the previous ${cutoff} seconds. -

- - - - - - - - - - - - - %for job in recent_jobs: - - %if job.history and job.history.user: - - %else: - - %endif - - - - <% - try: - inputs = ", ".join( [ '%s %s' % ( da.dataset.id, da.dataset.state ) for da in job.input_datasets ] ) - except: - inputs = 'Unable to determine inputs' - %> - - - - - - %endfor -
Job IDUserFinishedToolStateInputsCommand LineJob RunnerPID/Cluster ID
${job.id}${job.history.user.email|h}anonymous${finished[job.id]} ago${job.tool_id|h}${job.state}${inputs}${job.command_line|h}${job.job_runner_name|h}${job.job_runner_external_id|h}
-

-%else: -

There are no recently finished jobs to show with current cutoff time.
-

-%endif - -

-
-
- Update Jobs -
-
- -
- -
- -
-
- In seconds -
-
-
- -
-
-
-
- -
-

-

-
- Administrative Job Lock -
-
-
- - %if job_lock==True: -

Job dispatching is currently locked.

- - %else: -

Job dispatching is currently unlocked.

- - %endif -
-
- -
-
-
-
From 0f9e70b7b7c2265f3648ac7f64311ec43d6dc5a5 Mon Sep 17 00:00:00 2001 From: Sebastian Kapust Date: Sun, 21 Apr 2019 08:22:45 +0200 Subject: [PATCH 004/213] Fix js linting issue and page title. --- client/galaxy/scripts/components/admin/Jobs.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/galaxy/scripts/components/admin/Jobs.vue b/client/galaxy/scripts/components/admin/Jobs.vue index 5efa1cec465..ccfbabec0a8 100644 --- a/client/galaxy/scripts/components/admin/Jobs.vue +++ b/client/galaxy/scripts/components/admin/Jobs.vue @@ -1,7 +1,7 @@ -