From a7d0fc96289e6235e9d43802f3eb26da13b59f5f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 20:48:08 -0400 Subject: [PATCH 01/88] clean-fix-page-api - remove unused pages code, annotation param never set. --- client/galaxy/scripts/galaxy.pages.js | 2 -- lib/galaxy/webapps/galaxy/controllers/page.py | 33 +------------------ 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js index d9a1cf470fd..ecca493eb26 100644 --- a/client/galaxy/scripts/galaxy.pages.js +++ b/client/galaxy/scripts/galaxy.pages.js @@ -557,8 +557,6 @@ export default function pagesEditorOnload(renderedConfigs = {}) { data: { id: configs.page_id, content: editor.xhtml(), - annotations: JSON.stringify({}), - // annotations: JSON.stringify(annotations), _: "true" }, success: function() { diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 78b00076b42..669bdb24f9f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -685,7 +685,7 @@ class PageController(BaseUIController, SharableMixin, @web.expose @web.require_login() - def save(self, trans, id, content, annotations): + def save(self, trans, id, content): id = self.decode_id(id) page = trans.sa_session.query(model.Page).get(id) assert page.user == trans.user @@ -704,37 +704,6 @@ class PageController(BaseUIController, SharableMixin, page.latest_revision = page_revision page_revision.content = content - # Save annotations. - annotations = loads(annotations) - for annotation_dict in annotations: - item_id = self.decode_id(annotation_dict['item_id']) - item_class = self.get_class(annotation_dict['item_class']) - item = trans.sa_session.query(item_class).filter_by(id=item_id).first() - if not item: - raise RuntimeError("cannot find annotated item") - text = sanitize_html(annotation_dict['text']) - - # Add/update annotation. - if item_id and item_class and text: - # Get annotation association. - annotation_assoc_class = eval("model.%sAnnotationAssociation" % item_class.__name__) - annotation_assoc = trans.sa_session.query(annotation_assoc_class).filter_by(user=trans.get_user()) - if item_class == model.History.__class__: - annotation_assoc = annotation_assoc.filter_by(history=item) - elif item_class == model.HistoryDatasetAssociation.__class__: - annotation_assoc = annotation_assoc.filter_by(hda=item) - elif item_class == model.StoredWorkflow.__class__: - annotation_assoc = annotation_assoc.filter_by(stored_workflow=item) - elif item_class == model.WorkflowStep.__class__: - annotation_assoc = annotation_assoc.filter_by(workflow_step=item) - annotation_assoc = annotation_assoc.first() - if not annotation_assoc: - # Create association. - annotation_assoc = annotation_assoc_class() - item.annotations.append(annotation_assoc) - annotation_assoc.user = trans.get_user() - # Set annotation user text. - annotation_assoc.annotation = text trans.sa_session.flush() @web.expose From 31aa773066464cdcfd69a85b043b4d3229c7e9ae Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 21:16:20 -0400 Subject: [PATCH 02/88] clean-fix-page-api - page_revisions reuse manager util for checking security and decoding ids --- .../webapps/galaxy/api/page_revisions.py | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/page_revisions.py b/lib/galaxy/webapps/galaxy/api/page_revisions.py index faf2be37035..9d2233f282b 100644 --- a/lib/galaxy/webapps/galaxy/api/page_revisions.py +++ b/lib/galaxy/webapps/galaxy/api/page_revisions.py @@ -5,6 +5,7 @@ import logging from galaxy import exceptions from galaxy.model.item_attrs import UsesAnnotations +from galaxy.managers.base import get_object from galaxy.util.sanitize_html import sanitize_html from galaxy.web import expose_api from galaxy.web.base.controller import ( @@ -30,10 +31,8 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses :rtype: list :returns: dictionaries containing different revisions of the page """ - page = self._get_page(trans, page_id) - self._verify_page_ownership(trans, page) - - r = trans.sa_session.query(trans.app.model.PageRevision).filter_by(page_id=trans.security.decode_id(page_id)) + page = get_object(trans, page_id, 'Page', check_ownership=False, check_accessible=True) + r = trans.sa_session.query(trans.app.model.PageRevision).filter_by(page_id=page.id) out = [] for page in r: out.append(self.encode_all_ids(trans, page.to_dict(), True)) @@ -54,13 +53,12 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses :rtype: dictionary :returns: Dictionary with 'success' or 'error' element to indicate the result of the request """ + page = get_object(trans, page_id, 'Page', check_ownership=True) + content = payload.get("content", None) if not content: raise exceptions.ObjectAttributeMissingException("content undefined or empty") - page = self._get_page(trans, page_id) - self._verify_page_ownership(trans, page) - if 'title' in payload: title = payload['title'] else: @@ -79,17 +77,3 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses session.flush() return page_revision.to_dict(view="element") - - def _get_page(self, trans, page_id): - page = None - try: - page = trans.sa_session.query(trans.app.model.Page).get(trans.security.decode_id(page_id)) - except Exception: - pass - if not page: - raise exceptions.ObjectNotFound() - return page - - def _verify_page_ownership(self, trans, page): - if not self.security_check(trans, page, True, True): - raise exceptions.ItemOwnershipException() From 86d21ab5695deb193a794bf1026bf9bfd40ea86c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 22:25:24 -0400 Subject: [PATCH 03/88] clean-fix-page-api - pages API reuse manager util for checking security and decoding ids --- lib/galaxy/webapps/galaxy/api/pages.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py index a5a8ac3d75e..de69766d5ef 100644 --- a/lib/galaxy/webapps/galaxy/api/pages.py +++ b/lib/galaxy/webapps/galaxy/api/pages.py @@ -4,6 +4,7 @@ API for updating Galaxy Pages import logging from galaxy import exceptions +from galaxy.managers.base import get_object from galaxy.managers.pages import ( PageManager, PageSerializer @@ -128,7 +129,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati :rtype: dict :returns: Dictionary with 'success' or 'error' element to indicate the result of the request """ - page = self._get_page(trans, id) + page = get_object(trans, id, 'Page', check_ownership=True) # Mark a page as deleted page.deleted = True @@ -147,22 +148,7 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati :rtype: dict :returns: Dictionary return of the Page.to_dict call with the 'content' field populated by the most recent revision """ - page = self._get_page(trans, id) - self.security_check(trans, page, check_ownership=False, check_accessible=True) + page = get_object(trans, id, 'Page', check_ownership=False, check_accessible=True) rval = self.encode_all_ids(trans, page.to_dict(), True) rval['content'] = page.latest_revision.content return rval - - def _get_page(self, trans, id): # Fetches page object and verifies security. - try: - page = trans.sa_session.query(trans.app.model.Page).get(trans.security.decode_id(id)) - except Exception: - page = None - - if not page: - raise exceptions.ObjectNotFound() - - if page.user != trans.user and not trans.user_is_admin: - raise exceptions.ItemOwnershipException() - - return page From d324ad430e24bc5e1fdae946a91698abcff6b0e4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 17:39:49 -0400 Subject: [PATCH 04/88] clean-fix-page-api - refactor shared page creation code into manager. De-duplication between web and API controllers. --- lib/galaxy/managers/base.py | 7 +++ lib/galaxy/managers/pages.py | 47 ++++++++++++++- lib/galaxy/web/base/controller.py | 11 +--- .../webapps/galaxy/api/page_revisions.py | 5 ++ lib/galaxy/webapps/galaxy/api/pages.py | 37 +----------- lib/galaxy/webapps/galaxy/controllers/page.py | 57 ++++++------------- 6 files changed, 79 insertions(+), 85 deletions(-) diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index e06ca8de029..26b0fe2e01b 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -1128,3 +1128,10 @@ class ModelFilterParser(HasAModelManager): def raise_filter_err(self, attr, op, val, msg): raise exceptions.RequestParameterInvalidException(msg, column=attr, operation=op, val=val) + + +def is_valid_slug(slug): + """Returns true iff slug is valid.""" + + VALID_SLUG_RE = re.compile(r"^[a-z0-9\-]+$") + return VALID_SLUG_RE.match(slug) diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index 08f2c9f46fc..c8422cc638d 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -7,13 +7,16 @@ from within Galaxy. """ import logging -from galaxy import model +from galaxy import exceptions, model from galaxy.managers import sharable +from galaxy.managers.base import is_valid_slug +from galaxy.model.item_attrs import UsesAnnotations +from galaxy.util.sanitize_html import sanitize_html log = logging.getLogger(__name__) -class PageManager(sharable.SharableModelManager): +class PageManager(sharable.SharableModelManager, UsesAnnotations): """ """ @@ -35,6 +38,46 @@ class PageManager(sharable.SharableModelManager): """ pass + def create(self, trans, payload): + user = trans.get_user() + + if not payload.get("title", None): + raise exceptions.ObjectAttributeMissingException("Page name is required") + elif not payload.get("slug", None): + raise exceptions.ObjectAttributeMissingException("Page id is required") + elif not is_valid_slug(payload["slug"]): + raise exceptions.ObjectAttributeInvalidException("Page identifier must consist of only lowercase letters, numbers, and the '-' character") + elif trans.sa_session.query(trans.app.model.Page).filter_by(user=user, slug=payload["slug"], deleted=False).first(): + raise exceptions.DuplicatedSlugException("Page identifier must be unique") + + content = payload.get("content", "") + content = sanitize_html(content) + + # Create the new stored page + page = trans.app.model.Page() + page.title = payload['title'] + page.slug = payload['slug'] + page_annotation = payload.get("annotation", None) + if page_annotation is not None: + page_annotation = sanitize_html(page_annotation) + self.add_item_annotation(trans.sa_session, trans.get_user(), page, page_annotation) + + page.user = user + # And the first (empty) page revision + page_revision = trans.app.model.PageRevision() + page_revision.title = payload['title'] + page_revision.page = page + page.latest_revision = page_revision + page_revision.content = content + # Persist + session = trans.sa_session + session.add(page) + session.flush() + return page + + def save_revision(self, trans, page_id, payload): + pass + class PageSerializer(sharable.SharableModelSerializer): """ diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index da39c15233e..98468a54e3e 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -54,13 +54,6 @@ log = logging.getLogger(__name__) SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error" -def _is_valid_slug(slug): - """ Returns true if slug is valid. """ - - VALID_SLUG_RE = re.compile(r"^[a-z0-9\-]+$") - return VALID_SLUG_RE.match(slug) - - class BaseController(object): """ Base class for Galaxy web application controllers. @@ -1118,7 +1111,7 @@ class UsesVisualizationMixin(UsesLibraryMixinItems): title_err = slug_err = "" if not title: title_err = "visualization name is required" - elif slug and not _is_valid_slug(slug): + elif slug and not managers_base.is_valid_slug(slug): slug_err = "visualization identifier must consist of only lowercase letters, numbers, and the '-' character" elif slug and trans.sa_session.query(trans.model.Visualization).filter_by(user=user, slug=slug, deleted=False).first(): slug_err = "visualization identifier must be unique" @@ -1361,7 +1354,7 @@ class SharableMixin(object): def _is_valid_slug(self, slug): """ Returns true if slug is valid. """ - return _is_valid_slug(slug) + return managers_base.is_valid_slug(slug) @web.expose @web.require_login("modify Galaxy items") diff --git a/lib/galaxy/webapps/galaxy/api/page_revisions.py b/lib/galaxy/webapps/galaxy/api/page_revisions.py index 9d2233f282b..c3877dea5cb 100644 --- a/lib/galaxy/webapps/galaxy/api/page_revisions.py +++ b/lib/galaxy/webapps/galaxy/api/page_revisions.py @@ -6,6 +6,7 @@ import logging from galaxy import exceptions from galaxy.model.item_attrs import UsesAnnotations from galaxy.managers.base import get_object +from galaxy.managers.pages import PageManager from galaxy.util.sanitize_html import sanitize_html from galaxy.web import expose_api from galaxy.web.base.controller import ( @@ -19,6 +20,10 @@ log = logging.getLogger(__name__) class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotations, SharableMixin): + def __init__(self, app): + super(PageRevisionsController, self).__init__(app) + self.manager = PageManager(app) + @expose_api def index(self, trans, page_id, **kwd): """ diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py index de69766d5ef..29ddb522e96 100644 --- a/lib/galaxy/webapps/galaxy/api/pages.py +++ b/lib/galaxy/webapps/galaxy/api/pages.py @@ -10,7 +10,6 @@ from galaxy.managers.pages import ( PageSerializer ) from galaxy.model.item_attrs import UsesAnnotations -from galaxy.util.sanitize_html import sanitize_html from galaxy.web import expose_api from galaxy.web.base.controller import ( BaseAPIController, @@ -82,40 +81,8 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati :rtype: dict :returns: Dictionary return of the Page.to_dict call """ - user = trans.get_user() - - if not payload.get("title", None): - raise exceptions.ObjectAttributeMissingException("Page name is required") - elif not payload.get("slug", None): - raise exceptions.ObjectAttributeMissingException("Page id is required") - elif not self._is_valid_slug(payload["slug"]): - raise exceptions.ObjectAttributeInvalidException("Page identifier must consist of only lowercase letters, numbers, and the '-' character") - elif trans.sa_session.query(trans.app.model.Page).filter_by(user=user, slug=payload["slug"], deleted=False).first(): - raise exceptions.DuplicatedSlugException("Page slug must be unique") - - content = payload.get("content", "") - content = sanitize_html(content) - - # Create the new stored page - page = trans.app.model.Page() - page.title = payload['title'] - page.slug = payload['slug'] - page_annotation = sanitize_html(payload.get("annotation", "")) - self.add_item_annotation(trans.sa_session, trans.get_user(), page, page_annotation) - page.user = user - # And the first (empty) page revision - page_revision = trans.app.model.PageRevision() - page_revision.title = payload['title'] - page_revision.page = page - page.latest_revision = page_revision - page_revision.content = content - # Persist - session = trans.sa_session - session.add(page) - session.flush() - - rval = self.encode_all_ids(trans, page.to_dict(), True) - return rval + page = self.manager.create(trans, payload) + return self.encode_all_ids(trans, page.to_dict(), True) @expose_api def delete(self, trans, id, **kwd): diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 669bdb24f9f..92f9143e99c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -16,11 +16,15 @@ from sqlalchemy.orm import ( ) from galaxy import ( - managers, + exceptions, model, util, web ) +from galaxy.managers import base +from galaxy.managers.hdas import HDAManager +from galaxy.managers.histories import HistoryManager, HistorySerializer +from galaxy.managers.pages import PageManager from galaxy.model.item_attrs import UsesItemRatings from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html @@ -473,9 +477,10 @@ class PageController(BaseUIController, SharableMixin, def __init__(self, app): super(PageController, self).__init__(app) - self.history_manager = managers.histories.HistoryManager(app) - self.history_serializer = managers.histories.HistorySerializer(self.app) - self.hda_manager = managers.hdas.HDAManager(app) + self.page_manager = PageManager(app) + self.history_manager = HistoryManager(app) + self.history_serializer = HistorySerializer(self.app) + self.hda_manager = HDAManager(app) @web.expose @web.json @@ -541,37 +546,11 @@ class PageController(BaseUIController, SharableMixin, }] } else: - user = trans.get_user() - p_title = payload.get('title') - p_slug = payload.get('slug') - p_annotation = payload.get('annotation') - if not p_title: - return self.message_exception(trans, 'Please provide a page name is required.') - elif not p_slug: - return self.message_exception(trans, 'Please provide a unique identifier.') - elif not self._is_valid_slug(p_slug): - return self.message_exception(trans, 'Page identifier can only contain lowercase letters, numbers, and dashes (-).') - elif trans.sa_session.query(model.Page).filter_by(user=user, slug=p_slug, deleted=False).first(): - return self.message_exception(trans, 'Page id must be unique.') - else: - # Create the new stored page - p = model.Page() - p.title = p_title - p.slug = p_slug - p.user = user - if p_annotation: - p_annotation = sanitize_html(p_annotation) - self.add_item_annotation(trans.sa_session, user, p, p_annotation) - # And the first (empty) page revision - p_revision = model.PageRevision() - p_revision.title = p_title - p_revision.page = p - p.latest_revision = p_revision - p_revision.content = "" - # Persist - trans.sa_session.add(p) - trans.sa_session.flush() - return {'message': 'Page \'%s\' successfully created.' % p.title, 'status': 'success'} + try: + page = self.page_manager.create(trans, payload) + except exceptions.MessageException as e: + return self.message_exception(trans, str(e)) + return {'message': 'Page \'%s\' successfully created.' % page.title, 'status': 'success'} @web.legacy_expose_api @web.require_login("edit pages") @@ -958,20 +937,20 @@ def _placeholderRenderForSave(trans, item_class, item_id, encode=False): item_name = '' if item_class == 'History': history = trans.sa_session.query(trans.model.History).get(decoded_item_id) - history = managers.base.security_check(trans, history, False, True) + history = base.security_check(trans, history, False, True) item_name = history.name elif item_class == 'HistoryDatasetAssociation': hda = trans.sa_session.query(trans.model.HistoryDatasetAssociation).get(decoded_item_id) - hda_manager = managers.hdas.HDAManager(trans.app) + hda_manager = HDAManager(trans.app) hda = hda_manager.get_accessible(decoded_item_id, trans.user) item_name = hda.name elif item_class == 'StoredWorkflow': wf = trans.sa_session.query(trans.model.StoredWorkflow).get(decoded_item_id) - wf = managers.base.security_check(trans, wf, False, True) + wf = base.security_check(trans, wf, False, True) item_name = wf.name elif item_class == 'Visualization': visualization = trans.sa_session.query(trans.model.Visualization).get(decoded_item_id) - visualization = managers.base.security_check(trans, visualization, False, True) + visualization = base.security_check(trans, visualization, False, True) item_name = visualization.title class_shorthand = PAGE_CLASS_MAPPING[item_class] if encode: From ead233c0c1594d013f149eaf54773db7e29dcb8b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 21:32:38 -0400 Subject: [PATCH 05/88] clean-fix-page-api - move page revision creation to manager. --- lib/galaxy/managers/pages.py | 298 +++++++++++++++++- .../webapps/galaxy/api/page_revisions.py | 29 +- lib/galaxy/webapps/galaxy/controllers/page.py | 293 +---------------- scripts/pages_identifier_conversion.py | 4 +- 4 files changed, 308 insertions(+), 316 deletions(-) diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index c8422cc638d..b0b9c798f09 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -6,15 +6,51 @@ Pages are markup created and saved by users that can contain Galaxy objects from within Galaxy. """ import logging +import re + +from six.moves.html_entities import name2codepoint +from six.moves.html_parser import HTMLParser from galaxy import exceptions, model -from galaxy.managers import sharable -from galaxy.managers.base import is_valid_slug +from galaxy.managers import base, sharable +from galaxy.managers.hdas import HDAManager from galaxy.model.item_attrs import UsesAnnotations +from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html log = logging.getLogger(__name__) +# Copied from https://github.com/kurtmckee/feedparser +_cp1252 = { + 128: u'\u20ac', # euro sign + 130: u'\u201a', # single low-9 quotation mark + 131: u'\u0192', # latin small letter f with hook + 132: u'\u201e', # double low-9 quotation mark + 133: u'\u2026', # horizontal ellipsis + 134: u'\u2020', # dagger + 135: u'\u2021', # double dagger + 136: u'\u02c6', # modifier letter circumflex accent + 137: u'\u2030', # per mille sign + 138: u'\u0160', # latin capital letter s with caron + 139: u'\u2039', # single left-pointing angle quotation mark + 140: u'\u0152', # latin capital ligature oe + 142: u'\u017d', # latin capital letter z with caron + 145: u'\u2018', # left single quotation mark + 146: u'\u2019', # right single quotation mark + 147: u'\u201c', # left double quotation mark + 148: u'\u201d', # right double quotation mark + 149: u'\u2022', # bullet + 150: u'\u2013', # en dash + 151: u'\u2014', # em dash + 152: u'\u02dc', # small tilde + 153: u'\u2122', # trade mark sign + 154: u'\u0161', # latin small letter s with caron + 155: u'\u203a', # single right-pointing angle quotation mark + 156: u'\u0153', # latin small ligature oe + 158: u'\u017e', # latin small letter z with caron + 159: u'\u0178', # latin capital letter y with diaeresis +} + class PageManager(sharable.SharableModelManager, UsesAnnotations): """ @@ -45,7 +81,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): raise exceptions.ObjectAttributeMissingException("Page name is required") elif not payload.get("slug", None): raise exceptions.ObjectAttributeMissingException("Page id is required") - elif not is_valid_slug(payload["slug"]): + elif not base.is_valid_slug(payload["slug"]): raise exceptions.ObjectAttributeInvalidException("Page identifier must consist of only lowercase letters, numbers, and the '-' character") elif trans.sa_session.query(trans.app.model.Page).filter_by(user=user, slug=payload["slug"], deleted=False).first(): raise exceptions.DuplicatedSlugException("Page identifier must be unique") @@ -75,8 +111,33 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): session.flush() return page - def save_revision(self, trans, page_id, payload): - pass + def save_new_revision(self, trans, page, payload): + # Assumes security has already been checked by caller. + content = payload.get("content", None) + if not content: + raise exceptions.ObjectAttributeMissingException("content undefined or empty") + + if 'title' in payload: + title = payload['title'] + else: + title = page.title + + content = sanitize_html(content) + processor = PageContentProcessor(trans, placeholderRenderForSave) + processor.feed(content) + # Output is string, so convert to unicode for saving. + content = unicodify(processor.output(), 'utf-8') + + page_revision = trans.app.model.PageRevision() + page_revision.title = title + page_revision.page = page + page.latest_revision = page_revision + page_revision.content = content + + # Persist + session = trans.sa_session + session.flush() + return page_revision class PageSerializer(sharable.SharableModelSerializer): @@ -116,3 +177,230 @@ class PageDeserializer(sharable.SharableModelDeserializer): self.deserializers.update({ }) self.deserializable_keyset.update(self.deserializers.keys()) + + +class PageContentProcessor(HTMLParser, object): + """ + Processes page content to produce HTML that is suitable for display. + For now, processor renders embedded objects. + """ + bare_ampersand = re.compile(r"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") + elements_no_end_tag = set([ + 'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame', + 'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param', + 'source', 'track', 'wbr' + ]) + + def __init__(self, trans, render_embed_html_fn): + HTMLParser.__init__(self) + self.trans = trans + self.ignore_content = False + self.num_open_tags_for_ignore = 0 + self.render_embed_html_fn = render_embed_html_fn + + def reset(self): + self.pieces = [] + HTMLParser.reset(self) + + def _shorttag_replace(self, match): + tag = match.group(1) + if tag in self.elements_no_end_tag: + return '<' + tag + ' />' + else: + return '<' + tag + '>' + + def feed(self, data): + data = re.compile(r'\s]+?)\s*/>', self._shorttag_replace, data) + data = data.replace(''', "'") + data = data.replace('"', '"') + HTMLParser.feed(self, data) + HTMLParser.close(self) + + def handle_starttag(self, tag, attrs): + """ + Called for each start tag + + attrs is a list of (attr, value) tuples, e.g. for 
,
+        tag='pre', attrs=[('class', 'screen')]
+        """
+
+        # If ignoring content, just increment tag count and ignore.
+        if self.ignore_content:
+            self.num_open_tags_for_ignore += 1
+            return
+
+        # Not ignoring tag; look for embedded content.
+        embedded_item = False
+        for attribute in attrs:
+            if (attribute[0] == "class") and ("embedded-item" in attribute[1].split(" ")):
+                embedded_item = True
+                break
+        # For embedded content, set ignore flag to ignore current content and add new content for embedded item.
+        if embedded_item:
+            # Set processing attributes to ignore content.
+            self.ignore_content = True
+            self.num_open_tags_for_ignore = 1
+
+            # Insert content for embedded element.
+            for attribute in attrs:
+                name = attribute[0]
+                if name == "id":
+                    # ID has form '-'
+                    item_class, item_id = attribute[1].split("-")
+                    embed_html = self.render_embed_html_fn(self.trans, item_class, item_id)
+                    self.pieces.append(embed_html)
+            return
+
+        # Default behavior: not ignoring and no embedded content.
+        uattrs = []
+        strattrs = ''
+        if attrs:
+            for key, value in attrs:
+                value = value.replace('>', '>').replace('<', '<').replace('"', '"')
+                value = self.bare_ampersand.sub("&", value)
+                uattrs.append((key, value))
+            strattrs = ''.join(' %s="%s"' % (k, v) for k, v in uattrs)
+        if tag in self.elements_no_end_tag:
+            self.pieces.append('<%s%s />' % (tag, strattrs))
+        else:
+            self.pieces.append('<%s%s>' % (tag, strattrs))
+
+    def handle_endtag(self, tag):
+        """
+        Called for each end tag
+
+        E.g. for
, tag will be 'pre' + """ + # If ignoring content, see if current tag is the end of content to ignore. + if self.ignore_content: + self.num_open_tags_for_ignore -= 1 + if self.num_open_tags_for_ignore == 0: + # Done ignoring content. + self.ignore_content = False + return + + # Default behavior: reconstruct the original end tag. + if tag not in self.elements_no_end_tag: + self.pieces.append("" % tag) + + def handle_charref(self, ref): + # called for each character reference, e.g. for ' ', ref will be '160' + # Reconstruct the original character reference. + ref = ref.lower() + if ref.startswith('x'): + value = int(ref[1:], 16) + else: + value = int(ref) + + if value in _cp1252: + self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:]) + else: + self.pieces.append('&#%s;' % ref) + + def handle_entityref(self, ref): + # called for each entity reference, e.g. for '©', ref will be 'copy' + # Reconstruct the original entity reference. + if ref in name2codepoint or ref == 'apos': + self.pieces.append('&%s;' % ref) + else: + self.pieces.append('&%s' % ref) + + def handle_data(self, text): + """ + Called for each block of plain text + + Called outside of any tag and not containing any character or entity + references. Store the original text verbatim. + """ + if self.ignore_content: + return + self.pieces.append(text) + + def handle_comment(self, text): + # called for each HTML comment, e.g. + # Reconstruct the original comment. + self.pieces.append('' % text) + + def handle_decl(self, text): + # called for the DOCTYPE, if present, e.g. + # + # Reconstruct original DOCTYPE + self.pieces.append('' % text) + + def handle_pi(self, text): + # called for each processing instruction, e.g. + # Reconstruct original processing instruction. + self.pieces.append('' % text) + + def output(self): + '''Return processed HTML as a single string''' + return ''.join(self.pieces) + + +PAGE_MAXRAW = 10**15 + + +def get_page_identifiers(item_id, app): + # Assume if item id is integer and less than 10**15, it's unencoded. + try: + decoded_id = int(item_id) + if decoded_id >= PAGE_MAXRAW: + raise ValueError("Identifier larger than maximum expected raw int, must be already encoded.") + encoded_id = app.security.encode_id(item_id) + except ValueError: + # It's an encoded id. + encoded_id = item_id + decoded_id = app.security.decode_id(item_id) + return (encoded_id, decoded_id) + + +# Utilities for encoding/decoding HTML content. +PLACEHOLDER_TEMPLATE = '''

Embedded Galaxy {class_shorthand} - '{item_name}'

[Do not edit this block; Galaxy will fill it in with the annotated {class_shorthand} when it is displayed]

''' + +# This is a mapping of the id portion of page contents to the cssclass/shortname. +PAGE_CLASS_MAPPING = { + 'History': 'History', + 'HistoryDatasetAssociation': 'Dataset', + 'StoredWorkflow': 'Workflow', + 'Visualization': 'Visualization' +} + + +def placeholderRenderForEdit(trans, item_class, item_id): + return placeholderRenderForSave(trans, item_class, item_id, encode=True) + + +def placeholderRenderForSave(trans, item_class, item_id, encode=False): + encoded_item_id, decoded_item_id = get_page_identifiers(item_id, trans.app) + item_name = '' + if item_class == 'History': + history = trans.sa_session.query(trans.model.History).get(decoded_item_id) + history = base.security_check(trans, history, False, True) + item_name = history.name + elif item_class == 'HistoryDatasetAssociation': + hda = trans.sa_session.query(trans.model.HistoryDatasetAssociation).get(decoded_item_id) + hda_manager = HDAManager(trans.app) + hda = hda_manager.get_accessible(decoded_item_id, trans.user) + item_name = hda.name + elif item_class == 'StoredWorkflow': + wf = trans.sa_session.query(trans.model.StoredWorkflow).get(decoded_item_id) + wf = base.security_check(trans, wf, False, True) + item_name = wf.name + elif item_class == 'Visualization': + visualization = trans.sa_session.query(trans.model.Visualization).get(decoded_item_id) + visualization = base.security_check(trans, visualization, False, True) + item_name = visualization.title + class_shorthand = PAGE_CLASS_MAPPING[item_class] + if encode: + item_id = encoded_item_id + else: + item_id = decoded_item_id + return PLACEHOLDER_TEMPLATE.format( + item_class=item_class, + class_shorthand=class_shorthand, + class_shorthand_lower=class_shorthand.lower(), + item_id=item_id, + item_name=item_name + ) diff --git a/lib/galaxy/webapps/galaxy/api/page_revisions.py b/lib/galaxy/webapps/galaxy/api/page_revisions.py index c3877dea5cb..5cee22441ca 100644 --- a/lib/galaxy/webapps/galaxy/api/page_revisions.py +++ b/lib/galaxy/webapps/galaxy/api/page_revisions.py @@ -3,11 +3,9 @@ API for updating Galaxy Pages """ import logging -from galaxy import exceptions -from galaxy.model.item_attrs import UsesAnnotations from galaxy.managers.base import get_object from galaxy.managers.pages import PageManager -from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.item_attrs import UsesAnnotations from galaxy.web import expose_api from galaxy.web.base.controller import ( BaseAPIController, @@ -59,26 +57,5 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses :returns: Dictionary with 'success' or 'error' element to indicate the result of the request """ page = get_object(trans, page_id, 'Page', check_ownership=True) - - content = payload.get("content", None) - if not content: - raise exceptions.ObjectAttributeMissingException("content undefined or empty") - - if 'title' in payload: - title = payload['title'] - else: - title = page.title - - content = sanitize_html(content) - - page_revision = trans.app.model.PageRevision() - page_revision.title = title - page_revision.page = page - page.latest_revision = page_revision - page_revision.content = content - - # Persist - session = trans.sa_session - session.flush() - - return page_revision.to_dict(view="element") + page_revision = self.manager.save_new_revision(trans, page, payload) + return self.encode_all_ids(trans, page_revision.to_dict(view="element"), True) diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 92f9143e99c..92de407430f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -1,9 +1,5 @@ -import re -from json import loads from markupsafe import escape -from six.moves.html_entities import name2codepoint -from six.moves.html_parser import HTMLParser from sqlalchemy import ( and_, desc, @@ -21,10 +17,14 @@ from galaxy import ( util, web ) -from galaxy.managers import base from galaxy.managers.hdas import HDAManager from galaxy.managers.histories import HistoryManager, HistorySerializer -from galaxy.managers.pages import PageManager +from galaxy.managers.pages import ( + get_page_identifiers, + PageContentProcessor, + PageManager, + placeholderRenderForEdit, +) from galaxy.model.item_attrs import UsesItemRatings from galaxy.util import unicodify from galaxy.util.sanitize_html import sanitize_html @@ -44,55 +44,6 @@ from galaxy.web.framework.helpers import ( ) -# Copied from https://github.com/kurtmckee/feedparser -_cp1252 = { - 128: u'\u20ac', # euro sign - 130: u'\u201a', # single low-9 quotation mark - 131: u'\u0192', # latin small letter f with hook - 132: u'\u201e', # double low-9 quotation mark - 133: u'\u2026', # horizontal ellipsis - 134: u'\u2020', # dagger - 135: u'\u2021', # double dagger - 136: u'\u02c6', # modifier letter circumflex accent - 137: u'\u2030', # per mille sign - 138: u'\u0160', # latin capital letter s with caron - 139: u'\u2039', # single left-pointing angle quotation mark - 140: u'\u0152', # latin capital ligature oe - 142: u'\u017d', # latin capital letter z with caron - 145: u'\u2018', # left single quotation mark - 146: u'\u2019', # right single quotation mark - 147: u'\u201c', # left double quotation mark - 148: u'\u201d', # right double quotation mark - 149: u'\u2022', # bullet - 150: u'\u2013', # en dash - 151: u'\u2014', # em dash - 152: u'\u02dc', # small tilde - 153: u'\u2122', # trade mark sign - 154: u'\u0161', # latin small letter s with caron - 155: u'\u203a', # single right-pointing angle quotation mark - 156: u'\u0153', # latin small ligature oe - 158: u'\u017e', # latin small letter z with caron - 159: u'\u0178', # latin capital letter y with diaeresis -} - - -PAGE_MAXRAW = 10**15 - - -def _get_page_identifiers(item_id, app): - # Assume if item id is integer and less than 10**15, it's unencoded. - try: - decoded_id = int(item_id) - if decoded_id >= PAGE_MAXRAW: - raise ValueError("Identifier larger than maximum expected raw int, must be already encoded.") - encoded_id = app.security.encode_id(item_id) - except ValueError: - # It's an encoded id. - encoded_id = item_id - decoded_id = app.security.decode_id(item_id) - return (encoded_id, decoded_id) - - def format_bool(b): if b: return "yes" @@ -304,166 +255,6 @@ class VisualizationSelectionGrid(ItemSelectionGrid): # Adapted from the _BaseHTMLProcessor class of https://github.com/kurtmckee/feedparser -class _PageContentProcessor(HTMLParser, object): - """ - Processes page content to produce HTML that is suitable for display. - For now, processor renders embedded objects. - """ - bare_ampersand = re.compile(r"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") - elements_no_end_tag = set([ - 'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame', - 'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param', - 'source', 'track', 'wbr' - ]) - - def __init__(self, trans, render_embed_html_fn): - HTMLParser.__init__(self) - self.trans = trans - self.ignore_content = False - self.num_open_tags_for_ignore = 0 - self.render_embed_html_fn = render_embed_html_fn - - def reset(self): - self.pieces = [] - HTMLParser.reset(self) - - def _shorttag_replace(self, match): - tag = match.group(1) - if tag in self.elements_no_end_tag: - return '<' + tag + ' />' - else: - return '<' + tag + '>' - - def feed(self, data): - data = re.compile(r'\s]+?)\s*/>', self._shorttag_replace, data) - data = data.replace(''', "'") - data = data.replace('"', '"') - HTMLParser.feed(self, data) - HTMLParser.close(self) - - def handle_starttag(self, tag, attrs): - """ - Called for each start tag - - attrs is a list of (attr, value) tuples, e.g. for 
,
-        tag='pre', attrs=[('class', 'screen')]
-        """
-
-        # If ignoring content, just increment tag count and ignore.
-        if self.ignore_content:
-            self.num_open_tags_for_ignore += 1
-            return
-
-        # Not ignoring tag; look for embedded content.
-        embedded_item = False
-        for attribute in attrs:
-            if (attribute[0] == "class") and ("embedded-item" in attribute[1].split(" ")):
-                embedded_item = True
-                break
-        # For embedded content, set ignore flag to ignore current content and add new content for embedded item.
-        if embedded_item:
-            # Set processing attributes to ignore content.
-            self.ignore_content = True
-            self.num_open_tags_for_ignore = 1
-
-            # Insert content for embedded element.
-            for attribute in attrs:
-                name = attribute[0]
-                if name == "id":
-                    # ID has form '-'
-                    item_class, item_id = attribute[1].split("-")
-                    embed_html = self.render_embed_html_fn(self.trans, item_class, item_id)
-                    self.pieces.append(embed_html)
-            return
-
-        # Default behavior: not ignoring and no embedded content.
-        uattrs = []
-        strattrs = ''
-        if attrs:
-            for key, value in attrs:
-                value = value.replace('>', '>').replace('<', '<').replace('"', '"')
-                value = self.bare_ampersand.sub("&", value)
-                uattrs.append((key, value))
-            strattrs = ''.join(' %s="%s"' % (k, v) for k, v in uattrs)
-        if tag in self.elements_no_end_tag:
-            self.pieces.append('<%s%s />' % (tag, strattrs))
-        else:
-            self.pieces.append('<%s%s>' % (tag, strattrs))
-
-    def handle_endtag(self, tag):
-        """
-        Called for each end tag
-
-        E.g. for
, tag will be 'pre' - """ - # If ignoring content, see if current tag is the end of content to ignore. - if self.ignore_content: - self.num_open_tags_for_ignore -= 1 - if self.num_open_tags_for_ignore == 0: - # Done ignoring content. - self.ignore_content = False - return - - # Default behavior: reconstruct the original end tag. - if tag not in self.elements_no_end_tag: - self.pieces.append("" % tag) - - def handle_charref(self, ref): - # called for each character reference, e.g. for ' ', ref will be '160' - # Reconstruct the original character reference. - ref = ref.lower() - if ref.startswith('x'): - value = int(ref[1:], 16) - else: - value = int(ref) - - if value in _cp1252: - self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:]) - else: - self.pieces.append('&#%s;' % ref) - - def handle_entityref(self, ref): - # called for each entity reference, e.g. for '©', ref will be 'copy' - # Reconstruct the original entity reference. - if ref in name2codepoint or ref == 'apos': - self.pieces.append('&%s;' % ref) - else: - self.pieces.append('&%s' % ref) - - def handle_data(self, text): - """ - Called for each block of plain text - - Called outside of any tag and not containing any character or entity - references. Store the original text verbatim. - """ - if self.ignore_content: - return - self.pieces.append(text) - - def handle_comment(self, text): - # called for each HTML comment, e.g. - # Reconstruct the original comment. - self.pieces.append('' % text) - - def handle_decl(self, text): - # called for the DOCTYPE, if present, e.g. - # - # Reconstruct original DOCTYPE - self.pieces.append('' % text) - - def handle_pi(self, text): - # called for each processing instruction, e.g. - # Reconstruct original processing instruction. - self.pieces.append('' % text) - - def output(self): - '''Return processed HTML as a single string''' - return ''.join(self.pieces) - - class PageController(BaseUIController, SharableMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin, UsesItemRatings): @@ -617,7 +408,7 @@ class PageController(BaseUIController, SharableMixin, page = trans.sa_session.query(model.Page).get(id) assert page.user == trans.user content = page.latest_revision.content - processor = _PageContentProcessor(trans, _placeholderRenderForEdit) + processor = PageContentProcessor(trans, placeholderRenderForEdit) processor.feed(content) content = unicodify(processor.output(), 'utf-8') return trans.fill_template("page/editor.mako", page=page, content=content) @@ -668,22 +459,7 @@ class PageController(BaseUIController, SharableMixin, id = self.decode_id(id) page = trans.sa_session.query(model.Page).get(id) assert page.user == trans.user - - # Sanitize content - content = sanitize_html(content) - processor = _PageContentProcessor(trans, _placeholderRenderForSave) - processor.feed(content) - # Output is string, so convert to unicode for saving. - content = unicodify(processor.output(), 'utf-8') - - # Add a new revision to the page with the provided content. - page_revision = model.PageRevision() - page_revision.title = page.title - page_revision.page = page - page.latest_revision = page_revision - page_revision.content = content - - trans.sa_session.flush() + self.page_manager.save_new_revision(trans, page, {"content": content}) @web.expose @web.require_login() @@ -708,7 +484,7 @@ class PageController(BaseUIController, SharableMixin, self.security_check(trans, page, False, True) # Process page content. - processor = _PageContentProcessor(trans, self._get_embed_html) + processor = PageContentProcessor(trans, self._get_embed_html) processor.feed(page.latest_revision.content) # Output is string, so convert to unicode for display. page_content = unicodify(processor.output(), 'utf-8') @@ -890,7 +666,7 @@ class PageController(BaseUIController, SharableMixin, def _get_embed_html(self, trans, item_class, item_id): """ Returns HTML for embedding an item in a page. """ item_class = self.get_class(item_class) - encoded_id, decoded_id = _get_page_identifiers(item_id, trans.app) + encoded_id, decoded_id = get_page_identifiers(item_id, trans.app) if item_class == model.History: return self._get_embedded_history_html(trans, decoded_id) @@ -915,52 +691,3 @@ class PageController(BaseUIController, SharableMixin, elif item_class == model.Page: pass - - -PLACEHOLDER_TEMPLATE = '''

Embedded Galaxy {class_shorthand} - '{item_name}'

[Do not edit this block; Galaxy will fill it in with the annotated {class_shorthand} when it is displayed]

''' - -# This is a mapping of the id portion of page contents to the cssclass/shortname. -PAGE_CLASS_MAPPING = { - 'History': 'History', - 'HistoryDatasetAssociation': 'Dataset', - 'StoredWorkflow': 'Workflow', - 'Visualization': 'Visualization' -} - - -def _placeholderRenderForEdit(trans, item_class, item_id): - return _placeholderRenderForSave(trans, item_class, item_id, encode=True) - - -def _placeholderRenderForSave(trans, item_class, item_id, encode=False): - encoded_item_id, decoded_item_id = _get_page_identifiers(item_id, trans.app) - item_name = '' - if item_class == 'History': - history = trans.sa_session.query(trans.model.History).get(decoded_item_id) - history = base.security_check(trans, history, False, True) - item_name = history.name - elif item_class == 'HistoryDatasetAssociation': - hda = trans.sa_session.query(trans.model.HistoryDatasetAssociation).get(decoded_item_id) - hda_manager = HDAManager(trans.app) - hda = hda_manager.get_accessible(decoded_item_id, trans.user) - item_name = hda.name - elif item_class == 'StoredWorkflow': - wf = trans.sa_session.query(trans.model.StoredWorkflow).get(decoded_item_id) - wf = base.security_check(trans, wf, False, True) - item_name = wf.name - elif item_class == 'Visualization': - visualization = trans.sa_session.query(trans.model.Visualization).get(decoded_item_id) - visualization = base.security_check(trans, visualization, False, True) - item_name = visualization.title - class_shorthand = PAGE_CLASS_MAPPING[item_class] - if encode: - item_id = encoded_item_id - else: - item_id = decoded_item_id - return PLACEHOLDER_TEMPLATE.format( - item_class=item_class, - class_shorthand=class_shorthand, - class_shorthand_lower=class_shorthand.lower(), - item_id=item_id, - item_name=item_name - ) diff --git a/scripts/pages_identifier_conversion.py b/scripts/pages_identifier_conversion.py index ce914938f21..b9cec401071 100644 --- a/scripts/pages_identifier_conversion.py +++ b/scripts/pages_identifier_conversion.py @@ -9,12 +9,12 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa import galaxy import galaxy.app import galaxy.config +from galaxy.managers.pages import PageContentProcessor, placeholderRenderForSave from galaxy.objectstore import build_object_store_from_config from galaxy.security.idencoding import IdEncodingHelper from galaxy.util import unicodify from galaxy.util.bunch import Bunch from galaxy.util.script import app_properties_from_args, populate_config_args -from galaxy.webapps.galaxy.controllers.page import _PageContentProcessor, _placeholderRenderForSave def main(argv): @@ -37,7 +37,7 @@ def main(argv): mock_trans = Bunch(app=Bunch(security=security_helper), model=model, user_is_admin=lambda: True, sa_session=session) for p in pagerevs: try: - processor = _PageContentProcessor(mock_trans, _placeholderRenderForSave) + processor = PageContentProcessor(mock_trans, placeholderRenderForSave) processor.feed(p.content) newcontent = unicodify(processor.output(), 'utf-8') if p.content != newcontent: From ff00b800d34a06983106312cbd6673faf10c4cf9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 22:34:07 -0400 Subject: [PATCH 06/88] clean-fix-page-api - apply placeholderRenderForEdit to API. Was applied only in the controller previously. --- lib/galaxy/managers/pages.py | 8 ++++++++ lib/galaxy/webapps/galaxy/api/page_revisions.py | 8 ++++++-- lib/galaxy/webapps/galaxy/api/pages.py | 6 +++++- lib/galaxy/webapps/galaxy/controllers/page.py | 5 +---- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index b0b9c798f09..e940b572b32 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -139,6 +139,14 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): session.flush() return page_revision + def rewrite_content_for_export(self, trans, as_dict): + content = as_dict["content"] + processor = PageContentProcessor(trans, placeholderRenderForEdit) + processor.feed(content) + content = unicodify(processor.output(), 'utf-8') + as_dict["content"] = content + return as_dict + class PageSerializer(sharable.SharableModelSerializer): """ diff --git a/lib/galaxy/webapps/galaxy/api/page_revisions.py b/lib/galaxy/webapps/galaxy/api/page_revisions.py index 5cee22441ca..f240f6e9155 100644 --- a/lib/galaxy/webapps/galaxy/api/page_revisions.py +++ b/lib/galaxy/webapps/galaxy/api/page_revisions.py @@ -38,7 +38,9 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses r = trans.sa_session.query(trans.app.model.PageRevision).filter_by(page_id=page.id) out = [] for page in r: - out.append(self.encode_all_ids(trans, page.to_dict(), True)) + as_dict = self.encode_all_ids(trans, page.to_dict(), True) + self.manager.rewrite_content_for_export(trans, as_dict) + out.append(as_dict) return out @expose_api @@ -58,4 +60,6 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses """ page = get_object(trans, page_id, 'Page', check_ownership=True) page_revision = self.manager.save_new_revision(trans, page, payload) - return self.encode_all_ids(trans, page_revision.to_dict(view="element"), True) + rval = self.encode_all_ids(trans, page_revision.to_dict(view="element"), True) + self.manager.rewrite_content_for_export(trans, rval) + return rval diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py index 29ddb522e96..7f04c8a7204 100644 --- a/lib/galaxy/webapps/galaxy/api/pages.py +++ b/lib/galaxy/webapps/galaxy/api/pages.py @@ -82,7 +82,10 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati :returns: Dictionary return of the Page.to_dict call """ page = self.manager.create(trans, payload) - return self.encode_all_ids(trans, page.to_dict(), True) + rval = self.encode_all_ids(trans, page.to_dict(), True) + rval['content'] = page.latest_revision.content + self.manager.rewrite_content_for_export(trans, rval) + return rval @expose_api def delete(self, trans, id, **kwd): @@ -118,4 +121,5 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati page = get_object(trans, id, 'Page', check_ownership=False, check_accessible=True) rval = self.encode_all_ids(trans, page.to_dict(), True) rval['content'] = page.latest_revision.content + self.manager.rewrite_content_for_export(trans, rval) return rval diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index 92de407430f..af842df2a36 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -23,7 +23,6 @@ from galaxy.managers.pages import ( get_page_identifiers, PageContentProcessor, PageManager, - placeholderRenderForEdit, ) from galaxy.model.item_attrs import UsesItemRatings from galaxy.util import unicodify @@ -408,9 +407,7 @@ class PageController(BaseUIController, SharableMixin, page = trans.sa_session.query(model.Page).get(id) assert page.user == trans.user content = page.latest_revision.content - processor = PageContentProcessor(trans, placeholderRenderForEdit) - processor.feed(content) - content = unicodify(processor.output(), 'utf-8') + content = self.page_manager.rewrite_content_for_export(trans, {"content": content})["content"] return trans.fill_template("page/editor.mako", page=page, content=content) @web.expose From 5936478d20914b9b6b8bc571a3ee87cdafd12cc8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 22:50:20 -0400 Subject: [PATCH 07/88] clean-fix-page-api - apply placeholderRenderForSave to initial page save as well. --- lib/galaxy/managers/pages.py | 16 ++++++++++------ lib/galaxy/web/base/controller.py | 1 - 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index e940b572b32..4046b53e990 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -87,7 +87,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): raise exceptions.DuplicatedSlugException("Page identifier must be unique") content = payload.get("content", "") - content = sanitize_html(content) + content = self.rewrite_content_for_import(trans, content) # Create the new stored page page = trans.app.model.Page() @@ -122,11 +122,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): else: title = page.title - content = sanitize_html(content) - processor = PageContentProcessor(trans, placeholderRenderForSave) - processor.feed(content) - # Output is string, so convert to unicode for saving. - content = unicodify(processor.output(), 'utf-8') + content = self.rewrite_content_for_import(trans, content) page_revision = trans.app.model.PageRevision() page_revision.title = title @@ -139,6 +135,14 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): session.flush() return page_revision + def rewrite_content_for_import(self, trans, content): + content = sanitize_html(content) + processor = PageContentProcessor(trans, placeholderRenderForSave) + processor.feed(content) + # Output is string, so convert to unicode for saving. + content = unicodify(processor.output(), 'utf-8') + return content + def rewrite_content_for_export(self, trans, as_dict): content = as_dict["content"] processor = PageContentProcessor(trans, placeholderRenderForEdit) diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 98468a54e3e..49e209c8f5e 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -2,7 +2,6 @@ Contains functionality needed in every web interface """ import logging -import re from six import string_types from sqlalchemy import true From 98a4a8bd005236050623f86545c731ace1936798 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 26 May 2019 13:48:26 -0400 Subject: [PATCH 08/88] clean-fix-page-api - modern exception handling for embedded ID encoding --- lib/galaxy/managers/pages.py | 17 ++++++++++------ lib/galaxy/webapps/galaxy/api/pages.py | 1 - test/api/test_page_revisions.py | 6 +++--- test/api/test_pages.py | 28 +++++++++++++++++++++----- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index 4046b53e990..8d2053acce3 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -136,11 +136,16 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): return page_revision def rewrite_content_for_import(self, trans, content): - content = sanitize_html(content) - processor = PageContentProcessor(trans, placeholderRenderForSave) - processor.feed(content) - # Output is string, so convert to unicode for saving. - content = unicodify(processor.output(), 'utf-8') + try: + content = sanitize_html(content) + processor = PageContentProcessor(trans, placeholderRenderForSave) + processor.feed(content) + # Output is string, so convert to unicode for saving. + content = unicodify(processor.output(), 'utf-8') + except exceptions.MessageException: + raise + except Exception: + raise exceptions.RequestParameterInvalidException("problem with embedded HTML content [%s]" % content) return content def rewrite_content_for_export(self, trans, as_dict): @@ -364,7 +369,7 @@ def get_page_identifiers(item_id, app): except ValueError: # It's an encoded id. encoded_id = item_id - decoded_id = app.security.decode_id(item_id) + decoded_id = base.decode_id(app, item_id) return (encoded_id, decoded_id) diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py index 7f04c8a7204..c86090a29f0 100644 --- a/lib/galaxy/webapps/galaxy/api/pages.py +++ b/lib/galaxy/webapps/galaxy/api/pages.py @@ -3,7 +3,6 @@ API for updating Galaxy Pages """ import logging -from galaxy import exceptions from galaxy.managers.base import get_object from galaxy.managers.pages import ( PageManager, diff --git a/test/api/test_page_revisions.py b/test/api/test_page_revisions.py index 33d29d74fe6..fb89c90d680 100644 --- a/test/api/test_page_revisions.py +++ b/test/api/test_page_revisions.py @@ -28,8 +28,8 @@ class PageRevisionsApiTestCase(BasePageApiTestCase): revisions_json = revisions_response.json() assert len(revisions_json) == 2 # Original revision and new one - def test_404_if_index_unknown_page(self): + def test_malformed_id_if_index_unknown_page(self): revisions_url = "pages/%s/revisions" % self._random_key() revisions_response = self._get(revisions_url) - self._assert_status_code_is(revisions_response, 404) - self._assert_error_code_is(revisions_response, error_codes.USER_OBJECT_NOT_FOUND) + self._assert_status_code_is(revisions_response, 400) + self._assert_error_code_is(revisions_response, error_codes.MALFORMED_ID) diff --git a/test/api/test_pages.py b/test/api/test_pages.py index 6cabf05526a..4126fa0a46e 100644 --- a/test/api/test_pages.py +++ b/test/api/test_pages.py @@ -1,6 +1,7 @@ from requests import delete from base import api # noqa: I100,I202 +from base.populators import DatasetPopulator from galaxy.exceptions import error_codes # noqa: I201 @@ -53,7 +54,7 @@ class PageApiTestCase(BasePageApiTestCase): self._assert_error_code_is(page_response_2, error_codes.USER_SLUG_DUPLICATE) def test_page_requires_name(self): - page_request = self._test_page_payload() + page_request = self._test_page_payload(slug="requires-name") del page_request['title'] page_response = self._post("pages", page_request) self._assert_status_code_is(page_response, 400) @@ -70,10 +71,10 @@ class PageApiTestCase(BasePageApiTestCase): delete_response = delete(self._api_url("pages/%s" % response_json['id'], use_key=True)) self._assert_status_code_is(delete_response, 200) - def test_404_on_delete_unknown_page(self): + def test_400_on_delete_invalid_page_id(self): delete_response = delete(self._api_url("pages/%s" % self._random_key(), use_key=True)) - self._assert_status_code_is(delete_response, 404) - self._assert_error_code_is(delete_response, error_codes.USER_OBJECT_NOT_FOUND) + self._assert_status_code_is(delete_response, 400) + self._assert_error_code_is(delete_response, error_codes.MALFORMED_ID) def test_403_on_delete_unowned_page(self): page_response = self._create_valid_page_as("others_page@bx.psu.edu", "otherspage") @@ -81,6 +82,23 @@ class PageApiTestCase(BasePageApiTestCase): self._assert_status_code_is(delete_response, 403) self._assert_error_code_is(delete_response, error_codes.USER_DOES_NOT_OWN_ITEM) + def test_400_on_invalid_id_encoding(self): + page_request = self._test_page_payload(slug="invalid-id-encding") + page_request["content"] = '''

Page!

''' + page_response = self._post("pages", page_request) + self._assert_status_code_is(page_response, 400) + self._assert_error_code_is(page_response, error_codes.MALFORMED_ID) + + def test_400_on_invalid_embedded_content(self): + dataset_populator = DatasetPopulator(self.galaxy_interactor) + valid_id = dataset_populator.new_history() + page_request = self._test_page_payload(slug="invalid-id-encding") + page_request["content"] = '''

Page!

''' % valid_id + page_response = self._post("pages", page_request) + self._assert_status_code_is(page_response, 400) + self._assert_error_code_is(page_response, error_codes.USER_REQUEST_INVALID_PARAMETER) + assert "embedded HTML content" in page_response.content + def test_show(self): response_json = self._create_valid_page_with_slug("pagetoshow") show_response = self._get("pages/%s" % response_json['id']) @@ -95,7 +113,7 @@ class PageApiTestCase(BasePageApiTestCase): response_json = self._create_valid_page_as("others_page_show@bx.psu.edu", "otherspageshow") show_response = self._get("pages/%s" % response_json['id']) self._assert_status_code_is(show_response, 403) - self._assert_error_code_is(show_response, error_codes.USER_DOES_NOT_OWN_ITEM) + self._assert_error_code_is(show_response, error_codes.USER_CANNOT_ACCESS_ITEM) def _users_index_has_page_with_id(self, id): index_response = self._get("pages") From 2e3165cb7fcce6f81cea2c4f81737f1625d1db28 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 26 May 2019 14:34:56 -0400 Subject: [PATCH 09/88] clean-fix-page-api - integration API test case ensure DB contains unencoded IDs --- .../test_page_revision_json_encoding.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/integration/test_page_revision_json_encoding.py diff --git a/test/integration/test_page_revision_json_encoding.py b/test/integration/test_page_revision_json_encoding.py new file mode 100644 index 00000000000..f509c9b8c10 --- /dev/null +++ b/test/integration/test_page_revision_json_encoding.py @@ -0,0 +1,40 @@ +"""Test pages save JSON with unencoded IDs. + +Verifies the database doesn't get saved with encoded IDs (that would be bad because +the security parameter to encode IDs may be changed by admins). Test case also verifies +exported API values are encoded though. +""" + +from base import api_asserts +from base import integration_util +from base.populators import ( + DatasetPopulator, +) +from galaxy import model # noqa: I101,I201 + + +class PageJsonEncodingIntegrationTestCase(integration_util.IntegrationTestCase): + + def setUp(self): + super(PageJsonEncodingIntegrationTestCase, self).setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + + def test_page_encoding(self): + history_id = self.dataset_populator.new_history() + request = dict( + slug="mypage", + title="MY PAGE", + content='''

Page!

''' % history_id, + ) + page_response = self._post("pages", request) + api_asserts.assert_status_code_is_ok(page_response) + sa_session = self._app.model.context + page_revision = sa_session.query(model.PageRevision).all()[0] + assert '''id="History-1"''' in page_revision.content, page_revision.content + assert '''id="History-%s"''' % history_id not in page_revision.content, page_revision.content + + show_page_response = self._get("pages/%s" % page_response.json()["id"]) + api_asserts.assert_status_code_is_ok(show_page_response) + content = show_page_response.json()["content"] + assert '''id="History-1"''' not in content, content + assert '''id="History-%s"''' % history_id in content, content From 7a544aa31b9fe3cc7a7a6b163fdb3da3110339d9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 24 May 2019 19:47:18 -0400 Subject: [PATCH 10/88] Page-editor-as-component - initial Selenium test for pages + embedded objects. --- client/galaxy/scripts/galaxy.pages.js | 1 + client/galaxy/scripts/mvc/ui/ui-modal.js | 3 ++ test/galaxy_selenium/navigates_galaxy.py | 17 ++++++ test/galaxy_selenium/navigation.yml | 21 +++++++- test/selenium_tests/framework.py | 10 ++-- test/selenium_tests/test_histories_list.py | 36 ++++--------- test/selenium_tests/test_pages.py | 63 ++++++++++++++++++++++ 7 files changed, 118 insertions(+), 33 deletions(-) create mode 100644 test/selenium_tests/test_pages.py diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js index ecca493eb26..5bd284844d4 100644 --- a/client/galaxy/scripts/galaxy.pages.js +++ b/client/galaxy/scripts/galaxy.pages.js @@ -433,6 +433,7 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm ) ), closing_events: true, + extra_class: "pages-embed", buttons: { Embed: function() { // Make selected items accessible (importable) ? diff --git a/client/galaxy/scripts/mvc/ui/ui-modal.js b/client/galaxy/scripts/mvc/ui/ui-modal.js index 09c44b8dfc2..25dc0aa7679 100644 --- a/client/galaxy/scripts/mvc/ui/ui-modal.js +++ b/client/galaxy/scripts/mvc/ui/ui-modal.js @@ -81,6 +81,9 @@ export var View = Backbone.View.extend({ // link elements this.$header = this.$(".modal-header"); this.$dialog = this.$(".modal-dialog"); + if (this.options.extra_class) { + this.$dialog.addClass(this.options.extra_class); + } this.$body = this.$(".modal-body"); this.$footer = this.$(".modal-footer"); this.$backdrop = this.$(".modal-backdrop"); diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 7b9359ae4f7..a38b2a4fd80 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -331,6 +331,23 @@ class NavigatesGalaxy(HasDriver): raise self.prepend_timeout_message(e, message) return history_item_selector_state + def click_grid_popup_option(self, item_name, option_label): + item_button = None + grid = self.components.grids.body.wait_for_visible() + for row in grid.find_elements_by_tag_name('tr'): + name_cell = row.find_elements_by_tag_name('td')[1] + if name_cell.text == item_name: + item_button = name_cell + break + + if item_button is None: + raise AssertionError('Failed to find item with name [%s]' % item_name) + + popup_menu_button = item_button.find_element_by_css_selector('.dropdown-toggle') + popup_menu_button.click() + popup_option = self.driver.find_element_by_link_text(option_label) + popup_option.click() + def published_grid_search_for(self, search_term=None): return self._inline_search_for( '#input-free-text-search-filter', diff --git a/test/galaxy_selenium/navigation.yml b/test/galaxy_selenium/navigation.yml index 96b2fdd7e62..eae430b7eed 100644 --- a/test/galaxy_selenium/navigation.yml +++ b/test/galaxy_selenium/navigation.yml @@ -60,6 +60,7 @@ masthead: logout: 'Logout' custom_builds: 'Custom Builds' histories: 'Histories' + pages: 'Pages' # Shared data libraries: 'Data Libraries' @@ -201,6 +202,21 @@ histories: labels: unshare: 'Unshare' +pages: + selectors: + create: '.manage-table-actions .action-button' + submit: '#submit' + editor: + selectors: + wym_iframe: 'div.wym_iframe iframe' + wym_iframe_content: '.text-content' + save: '#save-button' + embed_button: '#embed-galaxy-object' + dataset_selector: '.saved-datasets' + embed_dialog_add_button: '.pages-embed .buttons #button-0' + + labels: + embed_dataset: 'Embed Dataset' login: selectors: @@ -365,9 +381,10 @@ libraries: table: '.dataset_table' table_rows: '.dataset_table table tbody tr' -published_grids: +grids: selectors: - search: '#input-free-text-search-filter' + body: '#grid-table-body' + free_text_search: '#input-free-text-search-filter' gies: jupyter: diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index 385e7469132..853c2a04ad1 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -92,10 +92,12 @@ def managed_history(f): f(self, *args, **kwds) finally: if "GALAXY_TEST_NO_CLEANUP" not in os.environ: - current_history_id = self.current_history_id() - self.dataset_populator.cancel_history_jobs(current_history_id) - self.api_delete("histories/%s" % current_history_id) - + try: + current_history_id = self.current_history_id() + self.dataset_populator.cancel_history_jobs(current_history_id) + self.api_delete("histories/%s" % current_history_id) + except Exception: + print("Faild to cleanup managed history, selenium connection corrupted somehow?") return func_wrapper diff --git a/test/selenium_tests/test_histories_list.py b/test/selenium_tests/test_histories_list.py index c250f4f1877..8f200f2640b 100644 --- a/test/selenium_tests/test_histories_list.py +++ b/test/selenium_tests/test_histories_list.py @@ -19,7 +19,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self._login() self.navigate_to_histories_page() self.screenshot("histories_saved_grid") - self.click_popup_option(self.history2_name, 'Switch') + self.click_grid_popup_option(self.history2_name, 'Switch') self.sleep_for(self.wait_types.UX_RENDER) @retry_assertion_during_transitions @@ -32,7 +32,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): def test_history_view(self): self._login() self.navigate_to_histories_page() - self.click_popup_option(self.history2_name, 'View') + self.click_grid_popup_option(self.history2_name, 'View') history_name = self.wait_for_selector('.name.editable-text') self.assertEqual(history_name.text, self.history2_name) @@ -42,7 +42,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.navigate_to_histories_page() # Publish the history - self.click_popup_option(self.history2_name, 'Share or Publish') + self.click_grid_popup_option(self.history2_name, 'Share or Publish') self.components.histories.sharing.make_accessible_and_publish.wait_for_and_click() self.navigate_to_histories_page() @@ -58,7 +58,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self._login() self.navigate_to_histories_page() - self.click_popup_option('Unnamed history', 'Rename') + self.click_grid_popup_option('Unnamed history', 'Rename') # Rename the history history_name_input = self.wait_for_selector('.ui-form-element input.ui-input') @@ -77,7 +77,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.navigate_to_histories_page() # Delete the history - self.click_popup_option(self.history2_name, 'Delete') + self.click_grid_popup_option(self.history2_name, 'Delete') self.assert_histories_in_grid([self.history2_name], False) @@ -86,7 +86,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.sleep_for(self.wait_types.UX_RENDER) # Restore the history - self.click_popup_option(self.history2_name, 'Undelete') + self.click_grid_popup_option(self.history2_name, 'Undelete') self.assert_grid_histories_are([]) self.select_filter('deleted', 'False') @@ -101,7 +101,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.navigate_to_histories_page() self.assert_histories_in_grid([self.history4_name]) - self.click_popup_option(self.history4_name, 'Delete Permanently') + self.click_grid_popup_option(self.history4_name, 'Delete Permanently') alert = self.driver.switch_to.alert alert.accept() @@ -164,15 +164,14 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self._login() self.navigate_to_histories_page() - input_selector = '#input-free-text-search-filter' - search_input = self.wait_for_selector(input_selector) + search_input = self.components.grids.free_text_search.wait_for_visible() search_input.send_keys(self.history2_name) self.send_enter(search_input) self.assert_grid_histories_are([self.history2_name]) self.unset_filter('free-text-search', self.history2_name) - search_input = self.wait_for_selector(input_selector) + search_input = self.components.grids.free_text_search.wait_for_visible() search_input.send_keys(self.history4_name) self.send_enter(search_input) @@ -299,23 +298,6 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): (filter_key, filter_value) self.wait_for_and_click_selector(filter_selector) - def click_popup_option(self, history_name, option_label): - history_menu_button = None - grid = self.wait_for_selector('#grid-table-body') - for row in grid.find_elements_by_tag_name('tr'): - name_cell = row.find_elements_by_tag_name('td')[1] - if name_cell.text == history_name: - history_menu_button = name_cell - break - - if history_menu_button is None: - raise AssertionError('Failed to find history with name [%s]' % history_name) - - popup_menu_button = history_menu_button.find_element_by_css_selector('.dropdown-toggle') - popup_menu_button.click() - popup_option = self.driver.find_element_by_link_text(option_label) - popup_option.click() - def get_history_tags_cell(self, history_name): tags_cell = None grid = self.wait_for_selector('#grid-table-body') diff --git a/test/selenium_tests/test_pages.py b/test/selenium_tests/test_pages.py new file mode 100644 index 00000000000..922ed4ce351 --- /dev/null +++ b/test/selenium_tests/test_pages.py @@ -0,0 +1,63 @@ +from .framework import ( + managed_history, + selenium_test, + SeleniumTestCase, +) + + +class PagesTestCase(SeleniumTestCase): + + ensure_registered = True + + @selenium_test + @managed_history + def test_simple_page_creation_edit_and_view(self): + # Upload a file to test embedded object stuff + test_path = self.get_filename("1.fasta") + self.perform_upload(test_path) + self.history_panel_wait_for_hid_ok(1) + self.navigate_to_pages() + self.screenshot("pages_grid") + + self.components.pages.create.wait_for_and_click() + name = self._get_random_name(prefix="page") + slug = self._get_random_name(prefix="pageslug") + self.tool_set_value("title", name) + self.tool_set_value("slug", slug) + self.screenshot("pages_create_form") + + # Sometimes 'submit' button not yet hooked up? + self.sleep_for(self.wait_types.UX_RENDER) + + self.components.pages.submit.wait_for_and_click() + + self.click_grid_popup_option(name, "Edit content") + self.components.pages.editor.wym_iframe.wait_for_visible() + self.screenshot("pages_editor_new") + self.driver.switch_to.frame(0) + try: + self.components.pages.editor.wym_iframe_content.wait_for_and_send_keys("moo\n\n\ncow\n\n") + finally: + self.driver.switch_to.default_content() + + self.components.pages.editor.embed_button.wait_for_and_click() + self.screenshot("pages_editor_embed_menu") + self.components.pages.editor.embed_dataset.wait_for_and_click() + saved_datasets_element = self.components.pages.editor.dataset_selector.wait_for_and_click() + self.screenshot("pages_editor_embed_dataset_dialog") + checkboxes = saved_datasets_element.find_elements_by_css_selector("input[type='checkbox']") + assert len(checkboxes) > 0 + checkboxes[0].click() + self.components.pages.editor.embed_dialog_add_button.wait_for_and_click() + + self.sleep_for(self.wait_types.UX_RENDER) + self.components.pages.editor.save.wait_for_and_click() + self.screenshot("pages_editor_saved") + self.home() + self.navigate_to_pages() + self.click_grid_popup_option(name, "View") + self.screenshot("pages_view_simple") + + def navigate_to_pages(self): + self.click_masthead_user() # Open masthead menu + self.components.masthead.pages.wait_for_and_click() From a71e77c40d0c94e0ab1bde9362850500b2040ece Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 14 May 2019 20:08:32 -0400 Subject: [PATCH 11/88] Page-editor-as-component - cleanup unused embed mako stuff. --- templates/embed_base.mako | 5 ----- templates/webapps/galaxy/dataset/embed.mako | 12 ------------ .../webapps/galaxy/visualization/embed.mako | 3 --- .../galaxy/visualization/embed_in_frame.mako | 4 ---- templates/webapps/galaxy/workflow/embed.mako | 18 ------------------ 5 files changed, 42 deletions(-) diff --git a/templates/embed_base.mako b/templates/embed_base.mako index 1e7c74b2622..a933919f76b 100644 --- a/templates/embed_base.mako +++ b/templates/embed_base.mako @@ -11,7 +11,6 @@ ${self.render_title( item )}
- ${self.render_summary_content( item, item_data )}
@@ -57,7 +56,3 @@ ## Use a hidden var to store the ajax URL for getting an item's content. - -## Methods to override to render summary content. -<%def name="render_summary_content( item, item_data )"> - diff --git a/templates/webapps/galaxy/dataset/embed.mako b/templates/webapps/galaxy/dataset/embed.mako index b31c04e2f27..74948f29527 100644 --- a/templates/webapps/galaxy/dataset/embed.mako +++ b/templates/webapps/galaxy/dataset/embed.mako @@ -1,8 +1,4 @@ <%inherit file="/embed_base.mako"/> -<%! - from galaxy.web.framework.helpers import iff -%> - <%def name="render_item_links( dataset )"> @@ -11,12 +7,4 @@ title="Import dataset" class="icon-button import"> - - - -<%def name="render_summary_content( dataset, data )"> -##
    -##
  • Format : ${dataset.extension} -## 
    ${dataset.peek}
    -##
diff --git a/templates/webapps/galaxy/visualization/embed.mako b/templates/webapps/galaxy/visualization/embed.mako index 242d6df7eea..9a6037dcfae 100644 --- a/templates/webapps/galaxy/visualization/embed.mako +++ b/templates/webapps/galaxy/visualization/embed.mako @@ -1,4 +1 @@ <%inherit file="/embed_base.mako"/> - -<%def name="render_summary_content( workflow, steps )"> - diff --git a/templates/webapps/galaxy/visualization/embed_in_frame.mako b/templates/webapps/galaxy/visualization/embed_in_frame.mako index b239bc1ba3a..2bd7d24c5ec 100644 --- a/templates/webapps/galaxy/visualization/embed_in_frame.mako +++ b/templates/webapps/galaxy/visualization/embed_in_frame.mako @@ -24,10 +24,6 @@
${item.annotation}
%endif
- ##
- ## currently, no summary content for visualization - ## could do the title or caption, whatever... - ##
diff --git a/templates/webapps/galaxy/workflow/embed.mako b/templates/webapps/galaxy/workflow/embed.mako index 2297b99c5fc..dee48db4490 100644 --- a/templates/webapps/galaxy/workflow/embed.mako +++ b/templates/webapps/galaxy/workflow/embed.mako @@ -1,24 +1,6 @@ <%inherit file="/embed_base.mako"/> -<%! - from galaxy.web.framework.helpers import iff -%> - <%def name="render_item_links( workflow )"> - ## FIXME: find and set appropriate icon for linking to workflow. - ${parent.render_item_links( workflow )} - -<%def name="render_summary_content( workflow, steps )"> - -##
    -## <% num_steps = len ( steps ) %> -##
  • ${num_steps} step${iff( num_steps != 1, "s", "" )} -##
  • Operations: ... -##
- From ba37ce53ee2c47eb0b0b888e9822758de0f33444 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 15 May 2019 09:20:41 -0400 Subject: [PATCH 12/88] Page-editor-as-component - refactor embedded mako JS toward reuse outside mako. --- client/galaxy/scripts/bundleEntries.js | 1 + client/galaxy/scripts/mvc/embedded-objects.js | 79 +++++++++++++++++++ templates/embed_base.mako | 2 +- templates/webapps/galaxy/page/display.mako | 73 +---------------- 4 files changed, 82 insertions(+), 73 deletions(-) create mode 100644 client/galaxy/scripts/mvc/embedded-objects.js diff --git a/client/galaxy/scripts/bundleEntries.js b/client/galaxy/scripts/bundleEntries.js index c666d664310..0390749fb83 100644 --- a/client/galaxy/scripts/bundleEntries.js +++ b/client/galaxy/scripts/bundleEntries.js @@ -79,6 +79,7 @@ export function multiHistory(options) { export { default as panels } from "layout/panel"; export { show_in_overlay, hide_modal, show_message, show_modal, Modal } from "layout/modal"; export { make_popupmenu, make_popup_menus } from "ui/popupmenu"; +export { render_embedded_items } from "mvc/embedded-objects"; export { default as async_save_text } from "utils/async-save-text"; // Previously "chart" diff --git a/client/galaxy/scripts/mvc/embedded-objects.js b/client/galaxy/scripts/mvc/embedded-objects.js new file mode 100644 index 00000000000..e89bbeb3e5f --- /dev/null +++ b/client/galaxy/scripts/mvc/embedded-objects.js @@ -0,0 +1,79 @@ +import $ from "jquery"; +import { make_popup_menus } from "ui/popupmenu"; + +export function render_embedded_items() { + $(".embedded-item").each(function() { + var container = $(this); + if (container.hasClass("history")) { + return; + } + //note: we can't do the same override for visualizations + // bc builtins (like trackster) need the handlers/ajax below to work. + // instead: (for registry visualizations) we'll clear the handlers below + // and add new ones (in embed_in_frame.mako) ...ugh. + + // Show embedded item. + var show_embedded_item = function() { + var ajax_url = container.find("input[type=hidden]").val(); + + // Only get item content if it's not already there. + var item_content = $.trim(container.find(".item-content").text()); + if (!item_content) { + $.ajax({ + type: "GET", + url: ajax_url, + error: function() { + alert("Getting item content failed."); + }, + success: function(item_content) { + container.find(".summary-content").hide("fast"); + container.find(".item-content").html(item_content); + container.find(".expanded-content").show("fast"); + container.find(".toggle-expand").hide(); + container.find(".toggle").show(); + + make_popup_menus(); + } + }); + } else { + container.find(".summary-content").hide("fast"); + container.find(".expanded-content").show("fast"); + container.find(".toggle-expand").hide(); + container.find(".toggle").show(); + } + }; + + // Hide embedded item. + var hide_embedded_item = function() { + container.find(".expanded-content").hide("fast"); + container.find(".summary-content").show("fast"); + container.find(".toggle").hide(); + container.find(".toggle-expand").show(); + }; + + // Setup toggle expand. + var toggle_expand = $(this).find(".toggle-expand"); + toggle_expand.click(function() { + show_embedded_item(); + return false; + }); + + // Setup toggle contract. + var toggle_contract = $(this).find(".toggle"); + toggle_contract.click(function() { + hide_embedded_item(); + return false; + }); + + // Setup toggle embed. + var toggle_embed = $(this).find(".toggle-embed"); + toggle_embed.click(function() { + if (container.find(".expanded-content").is(":visible")) { + hide_embedded_item(); + } else { + show_embedded_item(); + } + return false; + }); + }); +} diff --git a/templates/embed_base.mako b/templates/embed_base.mako index a933919f76b..163f293392a 100644 --- a/templates/embed_base.mako +++ b/templates/embed_base.mako @@ -41,7 +41,7 @@ display_href = h.url_for( controller=item_controller, action='display_by_username_and_slug', username=item_user.username, slug=item_slug ) %> diff --git a/templates/webapps/galaxy/page/display.mako b/templates/webapps/galaxy/page/display.mako index 1ed74810217..cd54a7500ea 100644 --- a/templates/webapps/galaxy/page/display.mako +++ b/templates/webapps/galaxy/page/display.mako @@ -7,80 +7,9 @@ From 6049515cee5e195d3ef36139dfa9899488106e34 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 15:07:11 -0400 Subject: [PATCH 13/88] Page-editor-as-component - generate properties client side. --- client/galaxy/scripts/galaxy.pages.js | 17 +++++++++++++++-- templates/webapps/galaxy/page/display.mako | 1 - templates/webapps/galaxy/page/editor.mako | 14 ++------------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js index 5bd284844d4..22336e69129 100644 --- a/client/galaxy/scripts/galaxy.pages.js +++ b/client/galaxy/scripts/galaxy.pages.js @@ -6,6 +6,7 @@ import { show_modal, hide_modal } from "layout/modal"; import { make_popupmenu } from "ui/popupmenu"; import { getGalaxyInstance } from "app"; import { buildConfig } from "utils/genericConfig"; +import { getAppRoot } from "onload/loadConfig"; // Built a generic config container for the properties that // are passed in from python then used (formerly) globally @@ -491,8 +492,20 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm } }; -export default function pagesEditorOnload(renderedConfigs = {}) { - setPageConfigs(renderedConfigs); +export default function pagesEditorOnload() { + const appRoot = getAppRoot(); + const pageId = $("[name=page_content]").attr("page_id"); + const pageConfigs = { + page_id: pageId, + page_list_url: `${appRoot}pages/list`, + list_objects_url: `${appRoot}page/LIST_ACTION`, + set_accessible_url: `${appRoot}ITEM_CONTROLLER/set_accessible_async`, + get_name_and_link_url: `${appRoot}ITEM_CONTROLLER/get_name_and_link_async?id=`, + editor_base_path: `${appRoot}static/wymeditor/`, + iframe_base_path: `${appRoot}static/wymeditor/iframe/galaxy/`, + save_url: `${appRoot}page/save` + } + setPageConfigs(pageConfigs); // Generic error handling $(document).ajaxError((e, x) => { diff --git a/templates/webapps/galaxy/page/display.mako b/templates/webapps/galaxy/page/display.mako index cd54a7500ea..5d3d29f613c 100644 --- a/templates/webapps/galaxy/page/display.mako +++ b/templates/webapps/galaxy/page/display.mako @@ -7,7 +7,6 @@ diff --git a/templates/webapps/galaxy/page/editor.mako b/templates/webapps/galaxy/page/editor.mako index 27e2eb0307f..01b07e20d4e 100644 --- a/templates/webapps/galaxy/page/editor.mako +++ b/templates/webapps/galaxy/page/editor.mako @@ -16,18 +16,8 @@ // Apparently pages() relies on these variables being defined // in window. config.addInitialization(function(){ - var pagesConfig = { - page_id: "${trans.security.encode_id(page.id)}", - page_list_url: '${h.url_for( controller='pages', action='list' )}', - list_objects_url: "${h.url_for(controller='page', action='LIST_ACTION' )}", - set_accessible_url: "${h.url_for( controller='ITEM_CONTROLLER', action='set_accessible_async' )}", - get_name_and_link_url: "${h.url_for( controller='ITEM_CONTROLLER', action='get_name_and_link_async' )}?id=", - editor_base_path: "${h.url_for('/static/wymeditor')}/", - iframe_base_path: "${h.url_for('/static/wymeditor/iframe/galaxy')}/", - save_url: "${h.url_for(controller='page', action='save' )}" - }; console.log("editor.mako, javascript_app", "define variables needed by galaxy.pages script"); - window.bundleEntries.pages(pagesConfig); + window.bundleEntries.pages(); }); @@ -55,7 +45,7 @@
- +
From 02832c6fbfb7fa9873492a55e74520392b7c4377 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 15:45:11 -0400 Subject: [PATCH 14/88] Page-editor-as-component - fetch content from pages API. --- client/galaxy/scripts/galaxy.pages.js | 28 +++++++++++++++++-- lib/galaxy/webapps/galaxy/controllers/page.py | 4 +-- templates/webapps/galaxy/page/editor.mako | 3 +- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js index 22336e69129..947749b1ad0 100644 --- a/client/galaxy/scripts/galaxy.pages.js +++ b/client/galaxy/scripts/galaxy.pages.js @@ -7,6 +7,8 @@ import { make_popupmenu } from "ui/popupmenu"; import { getGalaxyInstance } from "app"; import { buildConfig } from "utils/genericConfig"; import { getAppRoot } from "onload/loadConfig"; +import axios from "axios"; +import { Toast } from "ui/toast"; // Built a generic config container for the properties that // are passed in from python then used (formerly) globally @@ -492,9 +494,8 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm } }; -export default function pagesEditorOnload() { +function renderEditorWithContent(pageId, content) { const appRoot = getAppRoot(); - const pageId = $("[name=page_content]").attr("page_id"); const pageConfigs = { page_id: pageId, page_list_url: `${appRoot}pages/list`, @@ -514,8 +515,11 @@ export default function pagesEditorOnload() { show_modal("Server error", message, { "Ignore error": hide_modal }); return false; }); + $(".page-editor-content").append( + `` + ) // Create editor - $("[name=page_content]").wymeditor({ + $("[name=page_content]").val(content).wymeditor({ skin: "galaxy", basePath: configs.editor_base_path, iframeBasePath: configs.iframe_base_path, @@ -691,3 +695,21 @@ export default function pagesEditorOnload() { //} }); } + +export default function pagesEditorOnload() { + const pageId = $(".page-editor-content").attr("page_id"); + console.log(pageId); + axios + .get(`${getAppRoot()}api/pages/${pageId}`) + .then(response => { + renderEditorWithContent(pageId, response.data.content); + }) + .catch(e => { + const response = e.response; + if (typeof response.responseJSON !== "undefined") { + Toast.error(response.responseJSON.err_msg); + } else { + Toast.error("An error occurred."); + } + }); +} diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index af842df2a36..c5254697430 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -406,9 +406,7 @@ class PageController(BaseUIController, SharableMixin, id = self.decode_id(id) page = trans.sa_session.query(model.Page).get(id) assert page.user == trans.user - content = page.latest_revision.content - content = self.page_manager.rewrite_content_for_export(trans, {"content": content})["content"] - return trans.fill_template("page/editor.mako", page=page, content=content) + return trans.fill_template("page/editor.mako", page=page) @web.expose @web.require_login("use Galaxy pages") diff --git a/templates/webapps/galaxy/page/editor.mako b/templates/webapps/galaxy/page/editor.mako index 01b07e20d4e..7af1c89127b 100644 --- a/templates/webapps/galaxy/page/editor.mako +++ b/templates/webapps/galaxy/page/editor.mako @@ -44,8 +44,7 @@ -
- +
From 994a95e402eb23e954726c1f80b631f2583dd897 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 16:48:38 -0400 Subject: [PATCH 15/88] Page-editor-as-component - git mv to components to save commit history Not an atomic commit but cleaning the git history clean --- .../{galaxy.pages.js => components/PageEditor/PageEditorHtml.vue} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename client/galaxy/scripts/{galaxy.pages.js => components/PageEditor/PageEditorHtml.vue} (100%) diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue similarity index 100% rename from client/galaxy/scripts/galaxy.pages.js rename to client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue From 80e8ff3a1ed87aebb7576338d9d074225c1b54c6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 16:52:12 -0400 Subject: [PATCH 16/88] Page-editor-as-component - outline VueJS component for HTML editor. Call it PageEditorHtml to support a Markdown version at some point potentially. --- .../components/PageEditor/PageEditorHtml.vue | 153 ++++++++++-------- client/galaxy/scripts/galaxy.pages.js | 31 ++++ templates/webapps/galaxy/page/editor.mako | 24 +-- 3 files changed, 122 insertions(+), 86 deletions(-) create mode 100644 client/galaxy/scripts/galaxy.pages.js diff --git a/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue index 947749b1ad0..17f9ca73274 100644 --- a/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue +++ b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue @@ -1,3 +1,19 @@ + + + + + diff --git a/client/galaxy/scripts/galaxy.pages.js b/client/galaxy/scripts/galaxy.pages.js new file mode 100644 index 00000000000..c7cba88cf44 --- /dev/null +++ b/client/galaxy/scripts/galaxy.pages.js @@ -0,0 +1,31 @@ +import $ from "jquery"; +import axios from "axios"; +import { Toast } from "ui/toast"; +import { getAppRoot } from "onload/loadConfig"; +import PageEditorHtml from "components/PageEditor/PageEditorHtml"; +import Vue from "vue"; + +export default function pagesEditorOnload() { + const pageId = $("#page-editor-content").attr("page_id"); + axios + .get(`${getAppRoot()}api/pages/${pageId}`) + .then(response => { + const pageEditorHtmlInstance = Vue.extend(PageEditorHtml); + new pageEditorHtmlInstance({ + propsData: { + pageId: pageId, + content: response.data.content, + title: response.data.title + }, + el: "#page-editor-content" + }); + }) + .catch(e => { + const response = e.response; + if (typeof response.responseJSON !== "undefined") { + Toast.error(response.responseJSON.err_msg); + } else { + Toast.error("An error occurred."); + } + }); +} diff --git a/templates/webapps/galaxy/page/editor.mako b/templates/webapps/galaxy/page/editor.mako index 7af1c89127b..858774b54ef 100644 --- a/templates/webapps/galaxy/page/editor.mako +++ b/templates/webapps/galaxy/page/editor.mako @@ -22,29 +22,9 @@ -<%def name="stylesheets()"> - ${parent.stylesheets()} - ${h.css( "base", "embed_item" )} - - - <%def name="center_panel()"> -
-
- Page Editor: ${page.title | h} - -
-
- -
-
+ + From e96d0b493cd0bbb6a9937e402d3f738c96d85219 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 25 May 2019 17:55:12 -0400 Subject: [PATCH 17/88] Page-editor-as-component - refactor toward supporting multiple formats. Strong abstraction around "components/PageEditor" directory. --- .../components/PageEditor/PageEditor.vue | 33 +++++++++++++++++++ .../components/PageEditor/PageEditorHtml.vue | 18 ++-------- .../scripts/components/PageEditor/index.js | 1 + client/galaxy/scripts/galaxy.pages.js | 6 ++-- 4 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 client/galaxy/scripts/components/PageEditor/PageEditor.vue create mode 100644 client/galaxy/scripts/components/PageEditor/index.js diff --git a/client/galaxy/scripts/components/PageEditor/PageEditor.vue b/client/galaxy/scripts/components/PageEditor/PageEditor.vue new file mode 100644 index 00000000000..3309c7e7551 --- /dev/null +++ b/client/galaxy/scripts/components/PageEditor/PageEditor.vue @@ -0,0 +1,33 @@ + + + diff --git a/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue index 17f9ca73274..39a07756b88 100644 --- a/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue +++ b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue @@ -1,16 +1,7 @@ diff --git a/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue index 2bb20f69393..bce1497ac72 100644 --- a/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue +++ b/client/galaxy/scripts/components/PageEditor/PageEditorHtml.vue @@ -14,6 +14,7 @@ import { make_popupmenu } from "ui/popupmenu"; import { getGalaxyInstance } from "app"; import { buildConfig } from "utils/genericConfig"; import { getAppRoot } from "onload/loadConfig"; +import { save } from "./util"; // Built a generic config container for the properties that // are passed in from python then used (formerly) globally @@ -28,6 +29,8 @@ function setPageConfigs(incoming = {}) { var WYMeditor = window.WYMeditor; +var editor = null; + var CONTROLS = { // Item types. ITEM_HISTORY: "item_history", @@ -508,8 +511,7 @@ function renderEditorWithContent(pageId, content) { set_accessible_url: `${appRoot}ITEM_CONTROLLER/set_accessible_async`, get_name_and_link_url: `${appRoot}ITEM_CONTROLLER/get_name_and_link_async?id=`, editor_base_path: `${appRoot}static/wymeditor/`, - iframe_base_path: `${appRoot}static/wymeditor/iframe/galaxy/`, - save_url: `${appRoot}page/save` + iframe_base_path: `${appRoot}static/wymeditor/iframe/galaxy/` }; setPageConfigs(pageConfigs); @@ -568,30 +570,7 @@ function renderEditorWithContent(pageId, content) { ] }); // Get the editor object - var editor = $.wymeditors(0); - var save = callback => { - show_modal("Saving page", "progress"); - - // Do save. - $.ajax({ - url: configs.save_url, - type: "POST", - data: { - id: configs.page_id, - content: editor.xhtml(), - _: "true" - }, - success: function() { - callback(); - } - }); - }; - // Save button - $("#save-button").click(() => { - save(() => { - hide_modal(); - }); - }); + editor = $.wymeditors(0); // // Containers, Galaxy style @@ -678,12 +657,17 @@ export default { }, content: { type: String - }, + } }, created: function() { this.$nextTick(() => { renderEditorWithContent(this.pageId, this.content); }); + }, + methods: { + saveContent: function() { + save(this.pageId, editor.xhtml()); + } } }; diff --git a/client/galaxy/scripts/components/PageEditor/util.js b/client/galaxy/scripts/components/PageEditor/util.js new file mode 100644 index 00000000000..5339f5172f6 --- /dev/null +++ b/client/galaxy/scripts/components/PageEditor/util.js @@ -0,0 +1,24 @@ +import $ from "jquery"; +import _l from "utils/localization"; +import { getAppRoot } from "onload/loadConfig"; +import { show_modal, hide_modal } from "layout/modal"; + +export const save = (pageId, content) => { + console.log(`content is ${content}`); + + show_modal(_l("Saving page"), _l("progress")); + + // Do save. + $.ajax({ + url: `${getAppRoot()}page/save`, + type: "POST", + data: { + id: pageId, + content: content, + _: "true" + }, + success: function() { + hide_modal(); + } + }); +}; From f577541357d114c66e2abc71be88212d397c8ec5 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 3 Jun 2019 14:45:10 +0530 Subject: [PATCH 20/88] Filter all pykube queries by namespace --- lib/galaxy/jobs/runners/kubernetes.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 3594e044cb7..1c7748c51ea 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -468,7 +468,8 @@ class KubernetesJobRunner(AsynchronousJobRunner): marks the job for resubmission (resubmit logic is part of destinations). """ - pods = Pod.objects(self._pykube_api).filter(selector="app=%s" % job_state.job_id) + pods = Pod.objects(self._pykube_api).filter(selector="app=%s" % job_state.job_id, + namespace=self.runner_params['k8s_namespace']) pod = Pod(self._pykube_api, pods.response['items'][0]) if pod.obj['status']['phase'] == "Failed" and \ @@ -481,8 +482,9 @@ class KubernetesJobRunner(AsynchronousJobRunner): """Attempts to delete a dispatched job to the k8s cluster""" job = job_wrapper.get_job() try: - jobs = Job.objects(self._pykube_api).filter(selector="app=" + - self.__produce_unique_k8s_job_name(job.get_id_tag())) + jobs = Job.objects(self._pykube_api).filter( + selector="app=" + self.__produce_unique_k8s_job_name(job.get_id_tag()), + namespace=self.runner_params['k8s_namespace']) if len(jobs.response['items']) >= 0: job_to_delete = Job(self._pykube_api, jobs.response['items'][0]) job_to_delete.scale(replicas=0) From 91b6774f18df1dbf4a67476c8954f902bd4f6c59 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 4 Jun 2019 10:54:03 +0530 Subject: [PATCH 21/88] Added support for runAsUser and runAsGroup to k8srunner --- lib/galaxy/jobs/runners/kubernetes.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 3594e044cb7..f9b5d500644 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -54,6 +54,8 @@ class KubernetesJobRunner(AsynchronousJobRunner): k8s_job_api_version=dict(map=str, default="batch/v1"), k8s_supplemental_group_id=dict(map=str), k8s_pull_policy=dict(map=str, default="Default"), + k8s_run_as_user_id=dict(map=int, default=os.getuid()), + k8s_run_as_group_id=dict(map=int, default=app.config.gid), k8s_fs_group_id=dict(map=int), k8s_default_requests_cpu=dict(map=str, default=None), k8s_default_requests_memory=dict(map=str, default=None), @@ -237,17 +239,20 @@ class KubernetesJobRunner(AsynchronousJobRunner): } # TODO include other relevant elements that people might want to use from # TODO http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_podspec - - if self._supplemental_group and self._supplemental_group > 0: - k8s_spec_template["spec"]["securityContext"] = dict(supplementalGroups=[self._supplemental_group]) - if self._fs_group and self._fs_group > 0: - if "securityContext" in k8s_spec_template["spec"]: - k8s_spec_template["spec"]["securityContext"]["fsGroup"] = self._fs_group - else: - k8s_spec_template["spec"]["securityContext"] = dict(fsGroup=self._fs_group) - + k8s_spec_template["spec"]["securityContext"] = self.__get_k8s_security_context() return k8s_spec_template + def __get_k8s_security_context(self): + security_context = { + "runAsUser": int(self.runner_params["k8s_run_as_user_id"]), + "runAsGroup": int(self.runner_params["k8s_run_as_group_id"]) + } + if self._supplemental_group and self._supplemental_group > 0: + security_context["supplementalGroups"] = [self._supplemental_group] + if self._fs_group and self._fs_group > 0: + security_context["fsGroup"] = self._fs_group + return security_context + def __get_k8s_restart_policy(self, job_wrapper): """The default Kubernetes restart policy for Jobs""" return "Never" From 0a95f68f59a7f192cd7f257071ef1095f5ee8d39 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 15 May 2019 09:06:06 -0400 Subject: [PATCH 22/88] Unit test Galaxy packages during CI. All the same unit tests run in the context of a full Galaxy but this will ensure the tests also run with their dependencies as would be configured via pip-installed variants. This will catch dependency problems and ensure the package dependency structure remains correct. --- .circleci/config.yml | 9 +++++++++ packages/test.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100755 packages/test.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 01b3105991a..f46aa14dcea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -154,6 +154,13 @@ jobs: - run: sudo apt-get install -y libxml2-utils - *install_tox - run: tox -e validate_test_tools + test_galaxy_packages: + docker: + - image: circleci/python:2.7.15 + <<: *set_workdir + steps: + - *restore_repo_cache + - run: sh packages/test.sh check_py3_compatibility: docker: - image: circleci/python:2.7.15 @@ -202,6 +209,8 @@ workflows: <<: *requires_get_code - py35_first_startup: <<: *requires_get_code + - test_galaxy_packages: + <<: *requires_get_code - validate_test_tools: <<: *requires_get_code - check_py3_compatibility: diff --git a/packages/test.sh b/packages/test.sh new file mode 100755 index 00000000000..790d4bf67e8 --- /dev/null +++ b/packages/test.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -e + +# Change to packages directory. +cd "$(dirname "$0")" + +TEST_PYTHON=${TEST_PYTHON:-"python"} +TEST_ENV_DIR=${TEST_ENV_DIR:-`mktemp -d -t gxpkgtestenvXXXXXX`} + +virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" +. "${TEST_ENV_DIR}/bin/activate" +pip install pytest + +# ensure ordered by dependency dag +PACKAGE_DIRS=" + util + objectstore + data +" + +for package_dir in $PACKAGE_DIRS; do + cd "$package_dir" + python setup.py install + pytest --doctest-modules galaxy tests + cd .. +done From d7e08adeeb6a66bae3093b961c0f518c7110a2e9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 29 May 2019 15:45:58 -0400 Subject: [PATCH 23/88] Work on galaxy-util optional dependencies. --- packages/test.sh | 6 +++++- packages/util/requirements.txt | 1 - packages/util/setup.py | 4 ++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/test.sh b/packages/test.sh index 790d4bf67e8..d15d6648935 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -21,7 +21,11 @@ PACKAGE_DIRS=" for package_dir in $PACKAGE_DIRS; do cd "$package_dir" - python setup.py install + pip install -e . + if [ "$package_dir" = "util" ]; + then + pip install -e '.[template,jstree]' + fi pytest --doctest-modules galaxy tests cd .. done diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 3290e4ad496..e15e5db251e 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -1,7 +1,6 @@ bleach boltons bz2file; python_version < '3.3' -Cheetah3 docutils markupsafe packaging diff --git a/packages/util/setup.py b/packages/util/setup.py index ee54c34e225..1227150c7b1 100644 --- a/packages/util/setup.py +++ b/packages/util/setup.py @@ -80,6 +80,10 @@ setup( package_dir=PACKAGE_DIR, include_package_data=True, install_requires=requirements, + extras_require={ + 'template': ['future', 'Cheetah3'], + 'jstree': ['dictobj'], + }, license="AFL", zip_safe=False, keywords='galaxy', From a8b5b5b05c89869336f7a816635e9f1b1d8cc274 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 29 May 2019 16:33:37 -0400 Subject: [PATCH 24/88] Fix galaxy.util register_postfork_function references for non-web environments. --- lib/galaxy/util/monitors.py | 2 +- lib/galaxy/util/watcher.py | 2 +- lib/galaxy/util/web_compat.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 lib/galaxy/util/web_compat.py diff --git a/lib/galaxy/util/monitors.py b/lib/galaxy/util/monitors.py index 04a1f76359a..ee43d940ff3 100644 --- a/lib/galaxy/util/monitors.py +++ b/lib/galaxy/util/monitors.py @@ -3,8 +3,8 @@ from __future__ import absolute_import import logging import threading -from galaxy.web.stack import register_postfork_function from .sleeper import Sleeper +from .web_compat import register_postfork_function log = logging.getLogger(__name__) diff --git a/lib/galaxy/util/watcher.py b/lib/galaxy/util/watcher.py index e90141756cc..b8417b34fa1 100644 --- a/lib/galaxy/util/watcher.py +++ b/lib/galaxy/util/watcher.py @@ -21,7 +21,7 @@ except ImportError: can_watch = False from galaxy.util.hash_util import md5_hash_file -from galaxy.web.stack import register_postfork_function +from .web_compat import register_postfork_function log = logging.getLogger(__name__) diff --git a/lib/galaxy/util/web_compat.py b/lib/galaxy/util/web_compat.py new file mode 100644 index 00000000000..2c71d0e2b7f --- /dev/null +++ b/lib/galaxy/util/web_compat.py @@ -0,0 +1,10 @@ +"""Work around for gross circular dependency between galaxy.util and galaxy.web.stack. + +Provide a function that will delay to forking in a uwsgi environment but run immediately +otherwise. +""" +try: + from galaxy.web.stack import register_postfork_function +except ImportError: + def register_postfork_function(f, *args, **kwargs): + f(*args, **kwargs) From a6dc428aaafbd5ff23302e7011c81e177e3ed407 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 29 May 2019 16:49:02 -0400 Subject: [PATCH 25/88] Move none_like out of util to fix galaxy.util dependencies. --- lib/galaxy/{util => model}/none_like.py | 6 +----- lib/galaxy/tools/actions/__init__.py | 2 +- lib/galaxy/tools/evaluation.py | 2 +- lib/galaxy/tools/wrappers.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) rename lib/galaxy/{util => model}/none_like.py (78%) diff --git a/lib/galaxy/util/none_like.py b/lib/galaxy/model/none_like.py similarity index 78% rename from lib/galaxy/util/none_like.py rename to lib/galaxy/model/none_like.py index ff7c5de0475..293ecc5860c 100644 --- a/lib/galaxy/util/none_like.py +++ b/lib/galaxy/model/none_like.py @@ -2,7 +2,6 @@ Objects with No values """ -from galaxy.datatypes.registry import Registry from galaxy.model.metadata import MetadataCollection @@ -27,10 +26,7 @@ class NoneDataset(RecursiveNone): def __init__(self, datatypes_registry=None, ext='data', dbkey='?'): self.ext = self.extension = ext self.dbkey = dbkey - if datatypes_registry is None: - # Default Value Required for unit tests - datatypes_registry = Registry() - datatypes_registry.load_datatypes() + assert datatypes_registry is not None self.datatype = datatypes_registry.get_datatype_by_extension(ext) self._metadata = None self.metadata = MetadataCollection(self) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 3125f399355..cca6b198f67 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -10,12 +10,12 @@ from galaxy import model from galaxy.jobs.actions.post import ActionBox from galaxy.model import LibraryDatasetDatasetAssociation, WorkflowRequestInputParameter from galaxy.model.dataset_collections.builder import CollectionBuilder +from galaxy.model.none_like import NoneDataset from galaxy.objectstore import ObjectStorePopulator from galaxy.tools.parameters import update_dataset_ids from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue from galaxy.tools.parameters.wrapped import WrappedParameters from galaxy.util import ExecutionTimer -from galaxy.util.none_like import NoneDataset from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.web import url_for diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 162f0c906d1..01d25922a9b 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -8,6 +8,7 @@ from six import string_types from galaxy import model from galaxy.jobs.datasets import dataset_path_rewrites +from galaxy.model.none_like import NoneDataset from galaxy.tools import global_tool_errors from galaxy.tools.parameters import ( visit_input_values, @@ -38,7 +39,6 @@ from galaxy.util import ( unicodify, ) from galaxy.util.bunch import Bunch -from galaxy.util.none_like import NoneDataset from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.template import fill_template from galaxy.work.context import WorkRequestContext diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py index 8e23cb72585..9952f301308 100644 --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -6,8 +6,8 @@ from six import string_types, text_type from six.moves import shlex_quote from galaxy import exceptions +from galaxy.model.none_like import NoneDataset from galaxy.util import odict -from galaxy.util.none_like import NoneDataset from galaxy.util.object_wrapper import wrap_with_safe_string log = logging.getLogger(__name__) From bcbf7cbfed210885764a6c907e938c8d36040272 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 29 May 2019 21:56:44 -0400 Subject: [PATCH 26/88] Fix bx dependency for galaxy-data. --- packages/data/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 0c13610d96a..f2b5c80f06e 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -1,7 +1,7 @@ galaxy-objectstore galaxy-util bdbag -bx +bx-python h5py isa-rwval numpy From 07f17f66e9d2b565f7d89ca1f8a9945e4268ed1f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 29 May 2019 22:00:24 -0400 Subject: [PATCH 27/88] Redo import of unit test utilities for galaxy-data testing. --- packages/data/tests/unittest_utils | 1 - packages/data/tests/unittest_utils/__init__.py | 1 + packages/data/tests/unittest_utils/tempfilecache.py | 1 + packages/data/tests/unittest_utils/utility.py | 1 + 4 files changed, 3 insertions(+), 1 deletion(-) delete mode 120000 packages/data/tests/unittest_utils create mode 120000 packages/data/tests/unittest_utils/__init__.py create mode 120000 packages/data/tests/unittest_utils/tempfilecache.py create mode 120000 packages/data/tests/unittest_utils/utility.py diff --git a/packages/data/tests/unittest_utils b/packages/data/tests/unittest_utils deleted file mode 120000 index 6b6233367e6..00000000000 --- a/packages/data/tests/unittest_utils +++ /dev/null @@ -1 +0,0 @@ -../../../test/unit/unittest_utils \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/__init__.py b/packages/data/tests/unittest_utils/__init__.py new file mode 120000 index 00000000000..2115ad35783 --- /dev/null +++ b/packages/data/tests/unittest_utils/__init__.py @@ -0,0 +1 @@ +../../test/unit/unittest_utils/__init__.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/tempfilecache.py b/packages/data/tests/unittest_utils/tempfilecache.py new file mode 120000 index 00000000000..93fb83f1df7 --- /dev/null +++ b/packages/data/tests/unittest_utils/tempfilecache.py @@ -0,0 +1 @@ +../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/utility.py b/packages/data/tests/unittest_utils/utility.py new file mode 120000 index 00000000000..54dd4ca7a3b --- /dev/null +++ b/packages/data/tests/unittest_utils/utility.py @@ -0,0 +1 @@ +../../test/unit/unittest_utils/utility.py \ No newline at end of file From 6fa52930e9b568c68311b737fb395e8160e3e8a3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 30 May 2019 09:14:45 -0400 Subject: [PATCH 28/88] Modify packages being tested by test package stuff. --- packages/test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/test.sh b/packages/test.sh index d15d6648935..f1db04aff5c 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -16,8 +16,11 @@ pip install pytest PACKAGE_DIRS=" util objectstore - data + job_metrics + tool_util " +# 'data' package doesn't yet work. + for package_dir in $PACKAGE_DIRS; do cd "$package_dir" From ba187835111dbad7d87e63918fcdf13af0aba573 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 30 May 2019 09:33:10 -0400 Subject: [PATCH 29/88] Publish universal wheels for various packages. - Starting work on job-metrics 19.9.0.dev2 - Starting work on objectstore 19.9.0.dev2 - Starting work on util 19.9.0.dev3 - Starting work on tool-util 19.9.0.dev2 --- packages/job_metrics/galaxy/project_galaxy_job_metrics.py | 2 +- packages/objectstore/galaxy/project_galaxy_objectstore.py | 2 +- packages/tool_util/galaxy/project_galaxy_tool_util.py | 2 +- packages/util/galaxy/project_galaxy_util.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/job_metrics/galaxy/project_galaxy_job_metrics.py b/packages/job_metrics/galaxy/project_galaxy_job_metrics.py index 755ea7bf08c..5c5c5411ab8 100644 --- a/packages/job_metrics/galaxy/project_galaxy_job_metrics.py +++ b/packages/job_metrics/galaxy/project_galaxy_job_metrics.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev1' +__version__ = '19.9.0.dev2' PROJECT_NAME = "galaxy-job-metrics" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/objectstore/galaxy/project_galaxy_objectstore.py b/packages/objectstore/galaxy/project_galaxy_objectstore.py index 9aae9df3e85..825abc23062 100644 --- a/packages/objectstore/galaxy/project_galaxy_objectstore.py +++ b/packages/objectstore/galaxy/project_galaxy_objectstore.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev1' +__version__ = '19.9.0.dev2' PROJECT_NAME = "galaxy-objectstore" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index c6440bf685a..4f526958884 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev1' +__version__ = '19.9.0.dev2' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/packages/util/galaxy/project_galaxy_util.py b/packages/util/galaxy/project_galaxy_util.py index 1333aebadd2..c5a4d0ce99a 100644 --- a/packages/util/galaxy/project_galaxy_util.py +++ b/packages/util/galaxy/project_galaxy_util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '19.9.0.dev2' +__version__ = '19.9.0.dev3' PROJECT_NAME = "galaxy-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" From ed3434f85e3d340e6fd762efd5ffe91b4a6a4d27 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 30 May 2019 10:28:35 -0400 Subject: [PATCH 30/88] tool_util not yet working in CI testing... --- packages/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/test.sh b/packages/test.sh index f1db04aff5c..94e9c2f874d 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -17,9 +17,9 @@ PACKAGE_DIRS=" util objectstore job_metrics - tool_util " -# 'data' package doesn't yet work. +# tool_util seems to need containers... +# 'data' package doesn't yet work - quota, unit test problems, tool shed install database dependencies... for package_dir in $PACKAGE_DIRS; do From 3e2d992bd588d8d9ad49651ab77d65d5f30f13e9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 30 May 2019 10:40:30 -0400 Subject: [PATCH 31/88] Add galaxy-containers package... ... for galaxt-tool-util to depend on. --- packages/containers/HISTORY.rst | 12 +++ packages/containers/LICENSE | 1 + packages/containers/MANIFEST.in | 1 + packages/containers/Makefile | 1 + packages/containers/README.rst | 14 +++ packages/containers/dev-requirements.txt | 1 + packages/containers/galaxy/__init__.py | 1 + packages/containers/galaxy/containers | 1 + .../galaxy/project_galaxy_containers.py | 13 +++ packages/containers/requirements.txt | 3 + packages/containers/scripts | 1 + packages/containers/setup.cfg | 1 + packages/containers/setup.py | 102 ++++++++++++++++++ 13 files changed, 152 insertions(+) create mode 100644 packages/containers/HISTORY.rst create mode 120000 packages/containers/LICENSE create mode 100644 packages/containers/MANIFEST.in create mode 120000 packages/containers/Makefile create mode 100644 packages/containers/README.rst create mode 120000 packages/containers/dev-requirements.txt create mode 100644 packages/containers/galaxy/__init__.py create mode 120000 packages/containers/galaxy/containers create mode 100644 packages/containers/galaxy/project_galaxy_containers.py create mode 100644 packages/containers/requirements.txt create mode 120000 packages/containers/scripts create mode 120000 packages/containers/setup.cfg create mode 100644 packages/containers/setup.py diff --git a/packages/containers/HISTORY.rst b/packages/containers/HISTORY.rst new file mode 100644 index 00000000000..ec2f0e746e9 --- /dev/null +++ b/packages/containers/HISTORY.rst @@ -0,0 +1,12 @@ +.. :changelog: + +History +------- + +.. to_doc + +--------------------- +19.9.0.dev0 +--------------------- + +* Initial import from dev branch of Galaxy during 19.09 development cycle. diff --git a/packages/containers/LICENSE b/packages/containers/LICENSE new file mode 120000 index 00000000000..1ef648f64b3 --- /dev/null +++ b/packages/containers/LICENSE @@ -0,0 +1 @@ +../../LICENSE.txt \ No newline at end of file diff --git a/packages/containers/MANIFEST.in b/packages/containers/MANIFEST.in new file mode 100644 index 00000000000..c74cefe0942 --- /dev/null +++ b/packages/containers/MANIFEST.in @@ -0,0 +1 @@ +include *.rst LICENSE diff --git a/packages/containers/Makefile b/packages/containers/Makefile new file mode 120000 index 00000000000..37af8bae5ba --- /dev/null +++ b/packages/containers/Makefile @@ -0,0 +1 @@ +../package.Makefile \ No newline at end of file diff --git a/packages/containers/README.rst b/packages/containers/README.rst new file mode 100644 index 00000000000..ed360aeef60 --- /dev/null +++ b/packages/containers/README.rst @@ -0,0 +1,14 @@ + +.. image:: https://badge.fury.io/py/galaxy-containers.svg + :target: https://pypi.python.org/pypi/galaxy-containers/ + + +Overview +-------- + +The Galaxy_ containers module - interfaces for modeling and interacting with container backends (docker and docker swarm currently). + +* Free software: Academic Free License version 3.0 +* Code: https://github.com/galaxyproject/galaxy + +.. _Galaxy: http://galaxyproject.org/ diff --git a/packages/containers/dev-requirements.txt b/packages/containers/dev-requirements.txt new file mode 120000 index 00000000000..467b90d7a23 --- /dev/null +++ b/packages/containers/dev-requirements.txt @@ -0,0 +1 @@ +../package-dev-requirements.txt \ No newline at end of file diff --git a/packages/containers/galaxy/__init__.py b/packages/containers/galaxy/__init__.py new file mode 100644 index 00000000000..69e3be50dac --- /dev/null +++ b/packages/containers/galaxy/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/packages/containers/galaxy/containers b/packages/containers/galaxy/containers new file mode 120000 index 00000000000..bc2353d5e12 --- /dev/null +++ b/packages/containers/galaxy/containers @@ -0,0 +1 @@ +../../../lib/galaxy/containers \ No newline at end of file diff --git a/packages/containers/galaxy/project_galaxy_containers.py b/packages/containers/galaxy/project_galaxy_containers.py new file mode 100644 index 00000000000..cc927a1cdf7 --- /dev/null +++ b/packages/containers/galaxy/project_galaxy_containers.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +__version__ = '19.9.0.dev0' + +PROJECT_NAME = "galaxy-containers" +PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" +PROJECT_URL = "https://github.com/galaxyproject/galaxy" +PROJECT_AUTHOR = 'Galaxy Project and Community' +PROJECT_DESCRIPTION = 'Galaxy Container Modelling and Interaction Abstractions' +PROJECT_EMAIL = 'jmchilton@gmail.com' +RAW_CONTENT_URL = "https://raw.github.com/%s/%s/master/" % ( + PROJECT_USERAME, PROJECT_NAME +) diff --git a/packages/containers/requirements.txt b/packages/containers/requirements.txt new file mode 100644 index 00000000000..3a4655d7abd --- /dev/null +++ b/packages/containers/requirements.txt @@ -0,0 +1,3 @@ +galaxy-util +requests + diff --git a/packages/containers/scripts b/packages/containers/scripts new file mode 120000 index 00000000000..9aec9dc5a06 --- /dev/null +++ b/packages/containers/scripts @@ -0,0 +1 @@ +../build_scripts \ No newline at end of file diff --git a/packages/containers/setup.cfg b/packages/containers/setup.cfg new file mode 120000 index 00000000000..eb7cf09393f --- /dev/null +++ b/packages/containers/setup.cfg @@ -0,0 +1 @@ +../../setup.cfg \ No newline at end of file diff --git a/packages/containers/setup.py b/packages/containers/setup.py new file mode 100644 index 00000000000..4d4fb9caec1 --- /dev/null +++ b/packages/containers/setup.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import ast +import os +import re +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +SOURCE_DIR = "galaxy" + +_version_re = re.compile(r'__version__\s+=\s+(.*)') + +with open('%s/project_galaxy_containers.py' % SOURCE_DIR, 'rb') as f: + init_contents = f.read().decode('utf-8') + + def get_var(var_name): + pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + + version = get_var("__version__") + PROJECT_NAME = get_var("PROJECT_NAME") + PROJECT_URL = get_var("PROJECT_URL") + PROJECT_AUTHOR = get_var("PROJECT_AUTHOR") + PROJECT_EMAIL = get_var("PROJECT_EMAIL") + PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION") + +TEST_DIR = 'tests' +PACKAGES = [ + 'galaxy', + 'galaxy.containers', +] +ENTRY_POINTS = ''' + [console_scripts] +''' +PACKAGE_DATA = { + # Be sure to update MANIFEST.in for source dist. + 'galaxy': [ + ], +} +PACKAGE_DIR = { + SOURCE_DIR: SOURCE_DIR, +} + +readme = open('README.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') + +if os.path.exists("requirements.txt"): + requirements = open("requirements.txt").read().split("\n") +else: + # In tox, it will cover them anyway. + requirements = [] + + +test_requirements = [ + # TODO: put package test requirements here +] + + +setup( + name=PROJECT_NAME, + version=version, + description=PROJECT_DESCRIPTION, + long_description=readme + '\n\n' + history, + long_description_content_type='text/x-rst', + author=PROJECT_AUTHOR, + author_email=PROJECT_EMAIL, + url=PROJECT_URL, + packages=PACKAGES, + entry_points=ENTRY_POINTS, + package_data=PACKAGE_DATA, + package_dir=PACKAGE_DIR, + include_package_data=True, + install_requires=requirements, + extras_require={ + 'docker': ['docker'], + }, + license="AFL", + zip_safe=False, + keywords='galaxy', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Environment :: Console', + 'License :: OSI Approved :: Academic Free License (AFL)', + 'Operating System :: POSIX', + 'Topic :: Software Development', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Testing', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + test_suite=TEST_DIR, + tests_require=test_requirements +) From fb7efea6ba11674092c89c3e51c88f7cef1b44c9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 31 May 2019 11:51:50 -0400 Subject: [PATCH 32/88] Move tool XSD into galaxy-tool-util package. --- doc/Makefile | 4 ++-- lib/galaxy/{tools => tool_util}/xsd/LICENSE | 0 lib/galaxy/{tools => tool_util}/xsd/README.md | 0 lib/galaxy/{tools => tool_util}/xsd/galaxy.jxb | 0 lib/galaxy/{tools => tool_util}/xsd/galaxy.xsd | 0 packages/tool_util/MANIFEST.in | 4 +++- packages/tool_util/setup.py | 1 + scripts/validate_tools.sh | 2 +- 8 files changed, 7 insertions(+), 4 deletions(-) rename lib/galaxy/{tools => tool_util}/xsd/LICENSE (100%) rename lib/galaxy/{tools => tool_util}/xsd/README.md (100%) rename lib/galaxy/{tools => tool_util}/xsd/galaxy.jxb (100%) rename lib/galaxy/{tools => tool_util}/xsd/galaxy.xsd (100%) diff --git a/doc/Makefile b/doc/Makefile index 86166fce846..c4589e50479 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -42,8 +42,8 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " updaterst to update Sphinx RST files for lib/ to reflect code structure changes" -source/dev/schema.md: parse_gx_xsd.py schema_template.md ../lib/galaxy/tools/xsd/galaxy.xsd ## Build Github-flavored Markdown from Galaxy Tool XSD (expects lxml in environment) - python parse_gx_xsd.py schema_template.md ../lib/galaxy/tools/xsd/galaxy.xsd > $@ +source/dev/schema.md: parse_gx_xsd.py schema_template.md ../lib/galaxy/tool_util/xsd/galaxy.xsd ## Build Github-flavored Markdown from Galaxy Tool XSD (expects lxml in environment) + python parse_gx_xsd.py schema_template.md ../lib/galaxy/tool_util/xsd/galaxy.xsd > $@ source/api/api.rst: source/lib/galaxy.webapps.galaxy.api.rst printf "Galaxy API\n==========\n" > $@ diff --git a/lib/galaxy/tools/xsd/LICENSE b/lib/galaxy/tool_util/xsd/LICENSE similarity index 100% rename from lib/galaxy/tools/xsd/LICENSE rename to lib/galaxy/tool_util/xsd/LICENSE diff --git a/lib/galaxy/tools/xsd/README.md b/lib/galaxy/tool_util/xsd/README.md similarity index 100% rename from lib/galaxy/tools/xsd/README.md rename to lib/galaxy/tool_util/xsd/README.md diff --git a/lib/galaxy/tools/xsd/galaxy.jxb b/lib/galaxy/tool_util/xsd/galaxy.jxb similarity index 100% rename from lib/galaxy/tools/xsd/galaxy.jxb rename to lib/galaxy/tool_util/xsd/galaxy.jxb diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd similarity index 100% rename from lib/galaxy/tools/xsd/galaxy.xsd rename to lib/galaxy/tool_util/xsd/galaxy.xsd diff --git a/packages/tool_util/MANIFEST.in b/packages/tool_util/MANIFEST.in index 2bf148a5764..baf7e4b1ce2 100644 --- a/packages/tool_util/MANIFEST.in +++ b/packages/tool_util/MANIFEST.in @@ -1,2 +1,4 @@ include *.rst LICENSE - +include galaxy/tool_util/deps/mulled/invfile.lua +include galaxy/tool_util/deps/resolvers/default_conda_mapping.yml +include galaxy/tool_util/xsd/* diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py index 9645f9942fd..33bf10b7794 100644 --- a/packages/tool_util/setup.py +++ b/packages/tool_util/setup.py @@ -53,6 +53,7 @@ PACKAGE_DATA = { 'galaxy': [ 'tool_util/deps/mulled/invfile.lua', 'tool_util/deps/resolvers/default_conda_mapping.yml', + 'tool_util/xsd/galaxy.xsd', ], } PACKAGE_DIR = { diff --git a/scripts/validate_tools.sh b/scripts/validate_tools.sh index 16502e530d8..68d8bca2fad 100755 --- a/scripts/validate_tools.sh +++ b/scripts/validate_tools.sh @@ -10,7 +10,7 @@ then . "$GALAXY_VIRTUAL_ENV/bin/activate" fi -xsd_path="lib/galaxy/tools/xsd/galaxy.xsd" +xsd_path="lib/galaxy/tool_util/xsd/galaxy.xsd" err_tmp=`mktemp` count=0 From 5ad95a9b82899c407d373a25d52f4d57124b1102 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 31 May 2019 11:53:57 -0400 Subject: [PATCH 33/88] Add containers to package testing... --- .circleci/config.yml | 2 +- packages/test.sh | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f46aa14dcea..1a3cee44d75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,7 +160,7 @@ jobs: <<: *set_workdir steps: - *restore_repo_cache - - run: sh packages/test.sh + - run: bash packages/test.sh check_py3_compatibility: docker: - image: circleci/python:2.7.15 diff --git a/packages/test.sh b/packages/test.sh index 94e9c2f874d..5deb29bf919 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e @@ -13,22 +13,32 @@ virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" pip install pytest # ensure ordered by dependency dag -PACKAGE_DIRS=" +PACKAGE_DIRS=( util objectstore job_metrics -" -# tool_util seems to need containers... -# 'data' package doesn't yet work - quota, unit test problems, tool shed install database dependencies... + containers + tool_util + data +) +# containers has no tests, tool_util not yet working 100%, +# data has many problems quota, tool shed install database, etc.. +RUN_TESTS=(1 1 1 0 0 0 0) +for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do + package_dir=${PACKAGE_DIRS[$i]} + run_tests=${RUN_TESTS[$i]} -for package_dir in $PACKAGE_DIRS; do cd "$package_dir" pip install -e . if [ "$package_dir" = "util" ]; then pip install -e '.[template,jstree]' fi - pytest --doctest-modules galaxy tests + + if [[ "$run_tests" == "1" ]]; + then + pytest --doctest-modules galaxy tests + fi cd .. done From 2fb06e7cc0a0a3fb2f488f59c6e66af6da7194dc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 5 Jun 2019 09:08:55 -0400 Subject: [PATCH 34/88] Skip client build when validating test tools, unneeded. --- scripts/validate_tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate_tools.sh b/scripts/validate_tools.sh index 68d8bca2fad..11a52c2f0aa 100755 --- a/scripts/validate_tools.sh +++ b/scripts/validate_tools.sh @@ -2,7 +2,7 @@ cd `dirname $0`/.. -./scripts/common_startup.sh +GALAXY_SKIP_CLIENT_BUILD=1 ./scripts/common_startup.sh GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-.venv}" if [ -d "$GALAXY_VIRTUAL_ENV" ]; then From aa463ce070db39ffc0746cac8884aa43ba0c6426 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 5 Jun 2019 23:25:37 +0530 Subject: [PATCH 35/88] Default to none if run_as_user or group are not specified --- lib/galaxy/jobs/runners/kubernetes.py | 33 ++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index f9b5d500644..36d594333d7 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -54,8 +54,8 @@ class KubernetesJobRunner(AsynchronousJobRunner): k8s_job_api_version=dict(map=str, default="batch/v1"), k8s_supplemental_group_id=dict(map=str), k8s_pull_policy=dict(map=str, default="Default"), - k8s_run_as_user_id=dict(map=int, default=os.getuid()), - k8s_run_as_group_id=dict(map=int, default=app.config.gid), + k8s_run_as_user_id=dict(map=str, valid=lambda s: s == "$uid" or s.isdigit()), + k8s_run_as_group_id=dict(map=str, valid=lambda s: s == "$gid" or s.isdigit()), k8s_fs_group_id=dict(map=int), k8s_default_requests_cpu=dict(map=str, default=None), k8s_default_requests_memory=dict(map=str, default=None), @@ -77,6 +77,8 @@ class KubernetesJobRunner(AsynchronousJobRunner): self._galaxy_instance_id = self.__get_galaxy_instance_id() + self._run_as_user_id = self.__get_run_as_user_id() + self._run_as_group_id = self.__get_run_as_group_id() self._supplemental_group = self.__get_supplemental_group() self._fs_group = self.__get_fs_group() self._default_pull_policy = self.__get_pull_policy() @@ -170,6 +172,24 @@ class KubernetesJobRunner(AsynchronousJobRunner): return self.runner_params['k8s_pull_policy'] return None + def __get_run_as_user_id(self): + if "k8s_run_as_user_id" in self.runner_params: + run_as_user = self.runner_params["k8s_run_as_user_id"] + if run_as_user == "$uid": + return os.getuid() + else: + return int(self.runner_params["k8s_run_as_user_id"]) + return None + + def __get_run_as_group_id(self): + if "k8s_run_as_group_id" in self.runner_params: + run_as_group = self.runner_params["k8s_run_as_group_id"] + if run_as_group == "$gid": + return self.app.config.gid + else: + return int(self.runner_params["k8s_run_as_group_id"]) + return None + def __get_supplemental_group(self): if "k8s_supplemental_group_id" in self.runner_params: try: @@ -243,10 +263,11 @@ class KubernetesJobRunner(AsynchronousJobRunner): return k8s_spec_template def __get_k8s_security_context(self): - security_context = { - "runAsUser": int(self.runner_params["k8s_run_as_user_id"]), - "runAsGroup": int(self.runner_params["k8s_run_as_group_id"]) - } + security_context = {} + if self._run_as_user_id: + security_context["runAsUser"] = self._run_as_user_id + if self._run_as_group_id: + security_context["runAsGroup"] = self._run_as_group_id if self._supplemental_group and self._supplemental_group > 0: security_context["supplementalGroups"] = [self._supplemental_group] if self._fs_group and self._fs_group > 0: From 9ec952c0a7a6db510f01ea3b3e9459b2d2de11be Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 6 Jun 2019 15:58:15 +0100 Subject: [PATCH 36/88] Do not use dict iter methods for Python3 compatibility --- lib/galaxy/authnz/managers.py | 2 +- lib/galaxy/authnz/psa_authnz.py | 2 +- lib/galaxy/util/create_history_template.py | 2 +- lib/galaxy/visualization/plugins/registry.py | 2 +- lib/galaxy/webapps/galaxy/controllers/admin.py | 2 +- lib/galaxy/webapps/galaxy/controllers/data_manager.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index d31b5c52b23..c2211b19e9a 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -109,7 +109,7 @@ class AuthnzManager(object): def _unify_provider_name(self, provider): if provider.lower() in self.oidc_backends_config: return provider.lower() - for k, v in BACKENDS_NAME.iteritems(): + for k, v in BACKENDS_NAME.items(): if v == provider: return k.lower() diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index c4448ec3b27..06080f4bf13 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -82,7 +82,7 @@ DISCONNECT_PIPELINE = ( class PSAAuthnz(IdentityProvider): def __init__(self, provider, oidc_config, oidc_backend_config): self.config = {'provider': provider.lower()} - for key, value in oidc_config.iteritems(): + for key, value in oidc_config.items(): self.config[setting_name(key)] = value self.config[setting_name('USER_MODEL')] = 'models.User' diff --git a/lib/galaxy/util/create_history_template.py b/lib/galaxy/util/create_history_template.py index 24a2b8c0deb..290c8f4495f 100644 --- a/lib/galaxy/util/create_history_template.py +++ b/lib/galaxy/util/create_history_template.py @@ -142,7 +142,7 @@ def inputs_recursive(trans, input_params, param_values, depth=1, upgrade_message if upgrade_messages is None: upgrade_messages = {} - for input_index, input in enumerate(input_params.itervalues()): + for input_index, input in enumerate(input_params.values()): if input.name in param_values: if input.type == "repeat": for i in range(len(param_values[input.name])): diff --git a/lib/galaxy/visualization/plugins/registry.py b/lib/galaxy/visualization/plugins/registry.py index 88b10178aa5..8442484bad2 100644 --- a/lib/galaxy/visualization/plugins/registry.py +++ b/lib/galaxy/visualization/plugins/registry.py @@ -223,7 +223,7 @@ class VisualizationsRegistry(object): def get_plugins(self): result = [] - for plugin in self.plugins.itervalues(): + for plugin in self.plugins.values(): result.append(plugin.to_dict()) return sorted(result, key=lambda k: k.get('html')) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index b06bfa4841e..ab61b195479 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -558,7 +558,7 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP ) for data_table_elem_name, data_table in sorted_data_tables: - for filename, file_dict in data_table.filenames.iteritems(): + for filename, file_dict in data_table.filenames.items(): file_missing = ['file missing'] \ if not file_dict.get('found') else [] data.append({ diff --git a/lib/galaxy/webapps/galaxy/controllers/data_manager.py b/lib/galaxy/webapps/galaxy/controllers/data_manager.py index 493731321fc..fb5c47ce231 100644 --- a/lib/galaxy/webapps/galaxy/controllers/data_manager.py +++ b/lib/galaxy/webapps/galaxy/controllers/data_manager.py @@ -23,7 +23,7 @@ class DataManager(BaseUIController): message = kwd.get('message', '') status = kwd.get('status', 'info') data_managers = [] - for data_manager_id, data_manager in sorted(trans.app.data_managers.data_managers.iteritems(), + for data_manager_id, data_manager in sorted(trans.app.data_managers.data_managers.items(), key=lambda data_manager: data_manager[1].name): data_managers.append({'toolUrl': web.url_for(controller='root', tool_id=data_manager.tool.id), From 12451535293b4da28d7216ab1f7fed84eb7ea6ea Mon Sep 17 00:00:00 2001 From: Mason Date: Thu, 6 Jun 2019 10:17:36 -0700 Subject: [PATCH 37/88] Restored google analytics initialize inside mako templates --- templates/base.mako | 4 +++- templates/base/base_panels.mako | 2 +- templates/galaxy_client_app.mako | 17 +++++++++-------- templates/js-app.mako | 6 ++++-- templates/webapps/galaxy/galaxy.panels.mako | 2 +- templates/webapps/tool_shed/base_panels.mako | 2 +- 6 files changed, 19 insertions(+), 14 deletions(-) diff --git a/templates/base.mako b/templates/base.mako index 062d988ede5..8c6e3f2ba1d 100644 --- a/templates/base.mako +++ b/templates/base.mako @@ -65,7 +65,9 @@ ${ galaxy_client.load( app=self.js_app ) } ${ galaxy_client.config_sentry( app=self.js_app ) } - ${ galaxy_client.config_google_analytics( app=self.js_app ) } + %if self.js_app and self.js_app.config and self.js_app.config.ga_code: + ${ galaxy_client.config_google_analytics(self.js_app.config.ga_code) } + %endif %if not form_input_auto_focus is UNDEFINED and form_input_auto_focus: %if t.webapp.name == 'galaxy' and app.config.ga_code: - ${galaxy_client.config_google_analytics(app)} + ${galaxy_client.config_google_analytics(app.config.ga_code)} %endif diff --git a/templates/galaxy_client_app.mako b/templates/galaxy_client_app.mako index 4b4cb51d032..7c22f5d77fa 100644 --- a/templates/galaxy_client_app.mako +++ b/templates/galaxy_client_app.mako @@ -39,8 +39,6 @@ ${ h.dumps( dictionary, indent=( 2 if trans.debug else 0 ) ) } %if app and app.config: - %endif + %else: + console.warn("Missing google analytics code"); + %endif + diff --git a/templates/js-app.mako b/templates/js-app.mako index 1f25ff30a46..f606b272ba2 100644 --- a/templates/js-app.mako +++ b/templates/js-app.mako @@ -74,8 +74,10 @@ %endif - ## ${ galaxy_client.config_sentry(app)} - ## ${ galaxy_client.config_google_analytics(app)} + ${ galaxy_client.config_sentry(app) } + %if app.config.ga_code: + ${ galaxy_client.config_google_analytics(app.config.ga_code) } + %endif diff --git a/templates/webapps/galaxy/galaxy.panels.mako b/templates/webapps/galaxy/galaxy.panels.mako index 324ca1d25d8..39fc84ec0e7 100644 --- a/templates/webapps/galaxy/galaxy.panels.mako +++ b/templates/webapps/galaxy/galaxy.panels.mako @@ -225,7 +225,7 @@ ${ galaxy_client.load() } - ## ${ galaxy_client.config_sentry(app) } + ${ galaxy_client.config_sentry(app) } diff --git a/templates/webapps/tool_shed/base_panels.mako b/templates/webapps/tool_shed/base_panels.mako index 295f897679f..dd4c983abe4 100644 --- a/templates/webapps/tool_shed/base_panels.mako +++ b/templates/webapps/tool_shed/base_panels.mako @@ -46,7 +46,7 @@ <%def name="masthead()"> %if app.config.ga_code: - ${galaxy_client.config_google_analytics(app)} + ${ galaxy_client.config_google_analytics(app.config.ga_code)} %endif ## start main tag From aa25ff535599467c08bafbc435adccf0fac51e4a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 7 Jun 2019 00:04:53 +0100 Subject: [PATCH 38/88] Report a lint error if a citation is empty xref. https://github.com/galaxyproject/planemo/issues/929 Sync with https://github.com/galaxyproject/galaxy-lib/pull/151 --- lib/galaxy/tools/linters/citations.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tools/linters/citations.py b/lib/galaxy/tools/linters/citations.py index 95c03d078ab..23d8b63cdcb 100644 --- a/lib/galaxy/tools/linters/citations.py +++ b/lib/galaxy/tools/linters/citations.py @@ -21,12 +21,15 @@ def lint_citations(tool_xml, lint_ctx): for citation in citations[0]: if citation.tag != "citation": lint_ctx.warn("Unknown tag discovered in citations block [%s], will be ignored." % citation.tag) - if "type" in citation.attrib: - citation_type = citation.attrib.get("type") - if citation_type not in ["doi", "bibtex"]: - lint_ctx.warn("Unknown citation type discovered [%s], will be ignored.", citation_type) - else: - valid_citations += 1 + continue + citation_type = citation.attrib.get("type") + if citation_type not in ('bibtex', 'doi'): + lint_ctx.warn("Unknown citation type discovered [%s], will be ignored.", citation_type) + continue + if citation.text is None or not citation.text.strip(): + lint_ctx.error('Empty %s citation.' % citation_type) + continue + valid_citations += 1 if valid_citations > 0: lint_ctx.valid("Found %d likely valid citations.", valid_citations) From a69e03c9c7dee03df49d5419dec74071d0f2a22c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 7 Jun 2019 01:31:56 +0100 Subject: [PATCH 39/88] Add missing files from galaxy-lib --- lib/galaxy/tools/linters/cwl.py | 48 ++++++++++++++++++++++++++++++++ test/unit/tools/test_cwl_util.py | 25 +++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 lib/galaxy/tools/linters/cwl.py create mode 100644 test/unit/tools/test_cwl_util.py diff --git a/lib/galaxy/tools/linters/cwl.py b/lib/galaxy/tools/linters/cwl.py new file mode 100644 index 00000000000..58390b5ddd4 --- /dev/null +++ b/lib/galaxy/tools/linters/cwl.py @@ -0,0 +1,48 @@ +"""Linter for CWL tools.""" + +lint_tool_types = ["cwl"] + +from galaxy.tool_util.cwl.schema import schema_loader + + +def lint_cwl_validation(tool_source, lint_ctx): + """Determine in CWL tool validates against spec.""" + raw_reference = schema_loader.raw_process_reference(tool_source._source_path) + validation_exception = None + try: + schema_loader.process_definition(raw_reference) + except Exception as e: + validation_exception = e + if validation_exception: + lint_ctx.error("Failed to valdiate CWL artifact [%s]", validation_exception) + else: + lint_ctx.info("CWL appears to be valid.") + + +def lint_new_draft(tool_source, lint_ctx): + """Determine in CWL tool is valid, modern draft.""" + raw_reference = schema_loader.raw_process_reference(tool_source._source_path) + cwl_version = raw_reference.process_object.get("cwlVersion", None) + if cwl_version is None: + lint_ctx.error("CWL file does not contain a 'cwlVersion'") + if cwl_version not in ["v1.0"]: + lint_ctx.warn("CWL version [%s] is unknown, we recommend the v1.0 the stable release." % cwl_version) + else: + lint_ctx.info("Modern CWL version [%s]", cwl_version) + + +def lint_docker_image(tool_source, lint_ctx): + _, containers = tool_source.parse_requirements_and_containers() + if len(containers) == 0: + lint_ctx.warn("Tool does not specify a DockerPull source.") + else: + identifier = containers[0].identifier + lint_ctx.info("Tool will run in Docker image [%s]." % identifier) + + +def lint_description(tool_source, lint_ctx): + help = tool_source.parse_help() + if not help: + lint_ctx.warn("Description of tool is empty or absent.") + elif "TODO" in help: + lint_ctx.warn("Help contains TODO text.") diff --git a/test/unit/tools/test_cwl_util.py b/test/unit/tools/test_cwl_util.py new file mode 100644 index 00000000000..24c2ac95cf7 --- /dev/null +++ b/test/unit/tools/test_cwl_util.py @@ -0,0 +1,25 @@ +import tempfile + +from galaxy.tool_util.cwl.util import output_properties + + +def test_output_properties_in_memory(): + props = output_properties(content=b"hello world", basename="hello.txt") + assert props["basename"] == "hello.txt" + assert props["nameroot"] == "hello" + assert props["nameext"] == ".txt" + assert props["size"] == 11 + assert props["checksum"] == "sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed" + + +def test_output_properties_path(): + f = tempfile.NamedTemporaryFile(mode="w") + f.write("hello world") + f.flush() + + props = output_properties(path=f.name, basename="hello.txt") + assert props["basename"] == "hello.txt" + assert props["nameroot"] == "hello" + assert props["nameext"] == ".txt" + assert props["size"] == 11 + assert props["checksum"] == "sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed" From 39d265b907cd6af59cb97f947f3d702e2fbb1457 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 7 Jun 2019 07:19:25 +0530 Subject: [PATCH 40/88] Fix error in recognizing destination default container image --- lib/galaxy/tools/deps/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index 35229b4e631..10300a1666e 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -148,7 +148,7 @@ class ContainerFinder(object): key = "%s_image" % container_type if key in destination_info: return destination_info.get(key) - elif "%s_image_default" in destination_info: + elif "%s_image_default" % container_type in destination_info: return self.__build_container_id_from_parts(container_type, destination_info, mode="default") return None @@ -159,7 +159,7 @@ class ContainerFinder(object): # TODO: Right now this assumes all containers available when a # container type is - there should be more thought put into this. - # Checking which are availalbe - settings policies for what can be + # Checking which are available - settings policies for what can be # auto-fetched, etc.... return CONTAINER_CLASSES[container_type](container_id, self.app_info, tool_info, destination_info, job_info, container_description) From 8f5a35e58483542e26c1bbc62f43ab520453f97f Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 7 Jun 2019 15:13:14 -0400 Subject: [PATCH 41/88] I love RST --- doc/source/releases/19.05_announce.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/releases/19.05_announce.rst b/doc/source/releases/19.05_announce.rst index 955e75a83c0..557133feea8 100644 --- a/doc/source/releases/19.05_announce.rst +++ b/doc/source/releases/19.05_announce.rst @@ -17,7 +17,7 @@ Highlights user release notes instead. A huge thanks to `@erasche `__ for getting the ball rolling on this initiative. - Check out the `19.05 user release notes`__. + Check out the `19.05 user release notes `__. **Login and Registration Rewrite** The march toward replacing templated backend generated HTML with modern, reactive From 70730f673d0c19729f806e0c29bdef332824db85 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 7 Jun 2019 13:32:43 -0400 Subject: [PATCH 42/88] Allow declaring depedency resolvers right in Galaxy config, serialization. --- lib/galaxy/config.py | 11 ++- lib/galaxy/tool_util/deps/__init__.py | 83 +++++++++++++------ lib/galaxy/tool_util/deps/resolvers/conda.py | 2 +- .../tool_util/deps/resolvers/modules.py | 3 +- lib/galaxy/util/plugin_config.py | 13 ++- lib/galaxy/webapps/galaxy/config_schema.yml | 6 ++ test/unit/tools/test_tool_deps.py | 28 +++++-- 7 files changed, 106 insertions(+), 40 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 7a1fd5aa37a..f81a2293e95 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -50,7 +50,7 @@ PATH_DEFAULTS = dict( error_report_file=['config/error_report.yml', 'config/error_report.yml.sample'], oidc_config_file=['config/oidc_config.yml', 'config/oidc_config.yml.sample'], oidc_backends_config_file=['config/oidc_backends_config.yml', 'config/oidc_backends_config.yml.sample'], - dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'], + dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml', None], job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'], workflow_resource_params_file=['config/workflow_resource_params_conf.xml', 'workflow_resource_params_conf.xml'], migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'], @@ -142,6 +142,9 @@ def find_path(kwargs, var, root): path = kwargs.get(var) else: for default in defaults: + if default is None: + # if None is the final default - just return that. + return None if os.path.exists(resolve_path(default, root)): path = default break @@ -506,6 +509,7 @@ class Configuration(object): self.use_cached_dependency_manager = use_cached_dependency_manager self.tool_dependency_cache_dir = tool_dependency_cache_dir self.precache_dependencies = precache_dependencies + self.dependency_resolvers = kwargs.get("dependency_resolvers") # Deployers may either specify a complete list of mapping files or get the default for free and just # specify a local mapping file to adapt and extend the default one. if "conda_mapping_files" in kwargs: @@ -934,7 +938,10 @@ def parse_dependency_options(kwargs, root, dependency_resolvers_config_file): precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True')) else: tool_dependency_dir = None - use_tool_dependencies = os.path.exists(dependency_resolvers_config_file) + if dependency_resolvers_config_file is None: + use_tool_dependencies = bool(kwargs.get("dependency_resolvers", None)) + else: + use_tool_dependencies = os.path.exists(dependency_resolvers_config_file) tool_dependency_cache_dir = None precache_dependencies = False use_cached_dependency_manager = False diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index 1aab7f2e545..397a9a12faf 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -24,8 +24,6 @@ from .resolvers import ( ContainerDependency, NullDependency, ) -from .resolvers.conda import CondaDependencyResolver -from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver log = logging.getLogger(__name__) @@ -34,13 +32,26 @@ CONFIG_VAL_NOT_FOUND = object() def build_dependency_manager(config): - if getattr(config, "use_tool_dependencies", False): + config_dict = { + "use_tool_dependencies": getattr(config, "use_tool_dependencies", False), + "default_base_path": getattr(config, "tool_dependency_dir", None), + "conf_file": getattr(config, "dependency_resolvers_config_file", None), + "cache": getattr(config, "use_cached_dependency_manager", False), + "app_config": config, + "dependency_resolvers": getattr(config, "dependency_resolvers", None), + } + return build_dependency_manager_from_dict(config_dict) + + +def build_dependency_manager_from_dict(config_dict): + if config_dict.get("use_tool_dependencies", False): dependency_manager_kwds = { - 'default_base_path': config.tool_dependency_dir, - 'conf_file': config.dependency_resolvers_config_file, - 'app_config': config, + 'default_base_path': config_dict.get("default_base_path"), + 'conf_file': config_dict.get("conf_file"), + 'app_config': config_dict.get("app_config"), + "dependency_resolver_dicts": config_dict.get("dependency_resolvers"), } - if getattr(config, "use_cached_dependency_manager", False): + if config_dict.get("cache"): dependency_manager = CachedDependencyManager(**dependency_manager_kwds) else: dependency_manager = DependencyManager(**dependency_manager_kwds) @@ -61,8 +72,9 @@ class DependencyManager(object): and should each contain a file 'env.sh' which can be sourced to make the dependency available in the current shell environment. """ + cached = False - def __init__(self, default_base_path, conf_file=None, app_config={}): + def __init__(self, default_base_path, conf_file=None, app_config={}, dependency_resolver_dicts=None): """ Create a new dependency manager looking for packages under the paths listed in `base_paths`. The default base path is app.config.tool_dependency_dir. @@ -74,7 +86,13 @@ class DependencyManager(object): self.__app_config = app_config self.default_base_path = os.path.abspath(default_base_path) self.resolver_classes = self.__resolvers_dict() - self.dependency_resolvers = self.__build_dependency_resolvers(conf_file) + + plugin_source = None + if dependency_resolver_dicts is not None: + plugin_source = ('dict', dependency_resolver_dicts) + else: + plugin_source = self.__build_dependency_resolvers_plugin_source(conf_file) + self.dependency_resolvers = self.__parse_resolver_conf_plugins(plugin_source) self._enabled_container_types = [] self._destination_for_container_type = {} @@ -232,38 +250,50 @@ class DependencyManager(object): else: return NullDependency(name=name, version=version) - def __build_dependency_resolvers(self, conf_file): + def __build_dependency_resolvers_plugin_source(self, conf_file): if not conf_file: - return self.__default_dependency_resolvers() + return self.__default_dependency_resolvers_source() if not os.path.exists(conf_file): log.debug("Unable to find config file '%s'", conf_file) - return self.__default_dependency_resolvers() + return self.__default_dependency_resolvers_source() plugin_source = plugin_config.plugin_source_from_path(conf_file) - return self.__parse_resolver_conf_xml(plugin_source) + return plugin_source - def __default_dependency_resolvers(self): - return [ - ToolShedPackageDependencyResolver(self), - GalaxyPackageDependencyResolver(self), - CondaDependencyResolver(self), - GalaxyPackageDependencyResolver(self, versionless=True), - CondaDependencyResolver(self, versionless=True), - ] + def __default_dependency_resolvers_source(self): + return ('dict', [ + {"type": "tool_shed_packages"}, + {"type": "galaxy_packages"}, + {"type": "conda"}, + {"type": "galaxy_packages", "versionless": True}, + {"type": "conda", "versionless": True}, + ]) - def __parse_resolver_conf_xml(self, plugin_source): + def __parse_resolver_conf_plugins(self, plugin_source): """ """ extra_kwds = dict(dependency_manager=self) - return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds) + # Use either 'type' from YAML definition or 'resolver_type' from to_dict definition. + return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds, plugin_type_keys=['type', 'resolver_type']) def __resolvers_dict(self): import galaxy.tool_util.deps.resolvers return plugin_config.plugins_dict(galaxy.tool_util.deps.resolvers, 'resolver_type') + def to_dict(self): + return { + "cache": self.cached, + "use_tool_dependencies": True, + "default_base_path": self.default_base_path, + "dependency_resolvers": [m.to_dict() for m in self.dependency_resolvers], + "tool_dependency_cache_dir": getattr(self, "tool_dependency_cache_dir", None), + } + class CachedDependencyManager(DependencyManager): - def __init__(self, default_base_path, conf_file=None, app_config={}, tool_dependency_cache_dir=None): - super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config) + cached = True + + def __init__(self, default_base_path, **kwd): + super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, **kwd) self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir") def build_cache(self, requirements, **kwds): @@ -338,3 +368,6 @@ class NullDependencyManager(DependencyManager): def find_dep(self, name, version=None, type='package', **kwds): return NullDependency(version=version, name=name) + + def to_dict(self): + return {"use_tool_dependencies": False} diff --git a/lib/galaxy/tool_util/deps/resolvers/conda.py b/lib/galaxy/tool_util/deps/resolvers/conda.py index 28468ffb8ee..e7f4d3a25f0 100644 --- a/lib/galaxy/tool_util/deps/resolvers/conda.py +++ b/lib/galaxy/tool_util/deps/resolvers/conda.py @@ -65,7 +65,7 @@ log = logging.getLogger(__name__) class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): - dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install', 'auto_init'] resolver_type = "conda" config_options = { 'prefix': None, diff --git a/lib/galaxy/tool_util/deps/resolvers/modules.py b/lib/galaxy/tool_util/deps/resolvers/modules.py index fc7e17b62da..37ac3d70b8b 100644 --- a/lib/galaxy/tool_util/deps/resolvers/modules.py +++ b/lib/galaxy/tool_util/deps/resolvers/modules.py @@ -41,7 +41,7 @@ UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find mo class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver): - dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath'] + dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath', 'modulecmd', 'prefetch', 'default_indicator', 'find_by'] resolver_type = "modules" def __init__(self, dependency_manager, **kwds): @@ -50,6 +50,7 @@ class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver): self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') + self.find_by = find_by prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) self.modulecmd = kwds.get('modulecmd', DEFAULT_MODULECMD_PATH) self.modulepath = kwds.get('modulepath', self.__default_modulespath()) diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 1a4ce87f212..71fb410aab0 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -30,12 +30,12 @@ def plugins_dict(module, plugin_type_identifier): return plugin_dict -def load_plugins(plugins_dict, plugin_source, extra_kwds={}): +def load_plugins(plugins_dict, plugin_source, extra_kwds={}, plugin_type_keys=['type']): source_type, source = plugin_source if source_type == "xml": return __load_plugins_from_element(plugins_dict, source, extra_kwds) else: - return __load_plugins_from_dicts(plugins_dict, source, extra_kwds) + return __load_plugins_from_dicts(plugins_dict, source, extra_kwds, plugin_type_keys=plugin_type_keys) def __load_plugins_from_element(plugins_dict, plugins_element, extra_kwds): @@ -58,11 +58,16 @@ def __load_plugins_from_element(plugins_dict, plugins_element, extra_kwds): return plugins -def __load_plugins_from_dicts(plugins_dict, configs, extra_kwds): +def __load_plugins_from_dicts(plugins_dict, configs, extra_kwds, plugin_type_keys): plugins = [] for config in configs: - plugin_type = config["type"] + plugin_type = None + for plugin_type_key in plugin_type_keys: + if plugin_type_key in config: + plugin_type = config[plugin_type_key] + break + assert plugin_type is not None, "Could not determine plugin type for [%s]" % config plugin_kwds = config plugin_kwds.update(extra_kwds) plugin = plugins_dict[plugin_type](**plugin_kwds) diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 91b3482b4e1..bd393bec68a 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2364,6 +2364,12 @@ mapping: job_config: !include job_config_schema.yml + dependency_resolvers: + type: seq + desc: | + Rather than specifying a dependency_resolvers_config_file, the definition of the + resolvers to enable can be embedded into Galaxy's config with this option. + default_job_resubmission_condition: type: str required: false diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 5230466f802..0f609201a77 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -12,7 +12,7 @@ from shutil import rmtree from stat import S_IXUSR from subprocess import PIPE, Popen -from galaxy.tool_util.deps import DependencyManager +from galaxy.tool_util.deps import build_dependency_manager_from_dict, DependencyManager from galaxy.tool_util.deps.requirements import ( ToolRequirement, ToolRequirements @@ -23,6 +23,9 @@ from galaxy.tool_util.deps.resolvers.lmod import LmodDependency, LmodDependencyR from galaxy.tool_util.deps.resolvers.modules import ModuleDependency, ModuleDependencyResolver from galaxy.util.bunch import Bunch +# If true, test created DependencyManager objects by serializing out to json and re-constituting. +ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION = True + def test_tool_dependencies(): # Setup directories @@ -560,6 +563,13 @@ def test_config_module_defaults(): module_resolver = dependency_resolvers[0] assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" + with __parse_resolvers(''' +- type: modules + prefetch: false +''', extension=".yml") as dependency_resolvers: + module_resolver = dependency_resolvers[0] + assert module_resolver.module_checker.__class__.__name__ == "AvailModuleChecker" + def test_config_modulepath(): # Test reads and splits MODULEPATH if modulepath is not specified. @@ -623,23 +633,27 @@ def __environ(values, remove=[]): @contextmanager -def __parse_resolvers(xml_content): - with __dependency_manager(xml_content) as dm: +def __parse_resolvers(file_content, extension=".xml"): + with __dependency_manager(file_content, extension=extension) as dm: yield dm.dependency_resolvers @contextmanager -def __dependency_manager(xml_content): +def __dependency_manager(file_content, extension=".xml"): with __test_base_path() as base_path: - with tempfile.NamedTemporaryFile('w+') as tmp: - tmp.write(xml_content) + with tempfile.NamedTemporaryFile('w+', suffix=extension) as tmp: + tmp.write(file_content) tmp.flush() dm = __dependency_manager_for_base_path(default_base_path=base_path, conf_file=tmp.name) yield dm def __dependency_manager_for_base_path(default_base_path, conf_file=None): - return DependencyManager(default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False}) + dm = DependencyManager(default_base_path=default_base_path, conf_file=conf_file, app_config={"conda_auto_init": False}) + if ROUND_TRIP_TEST_DEPENDENCY_MANAGER_SERIALIZATION: + as_dict = dm.to_dict() + dm = build_dependency_manager_from_dict(as_dict) + return dm class _SimpleDependencyManager(object): From b5b1d82c99767ffcd64518033fc739f2ede7b59f Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 8 Jun 2019 11:37:31 -0700 Subject: [PATCH 43/88] Change cloudauthz version to 0.3.0. --- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index fd625aa8efb..56b6f3858f6 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -38,7 +38,7 @@ cffi==1.12.3 chardet==3.0.4 cheetah3==3.2.2 cliff==2.14.1 -cloudauthz==0.2.0 +cloudauthz==0.3.0 cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' From 96e15d25c54a4462f1b3a3fdb4b7b50d9aa01be0 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 8 Jun 2019 13:17:54 -0700 Subject: [PATCH 44/88] Result of running `make update-dependencies`. --- .../default/pinned-dev-requirements.txt | 32 +++++---- .../pipfiles/default/pinned-requirements.txt | 68 ++++++++++--------- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 282c185ca8d..d87cc2cc5ba 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -4,45 +4,47 @@ alabaster==0.7.12 argh==0.26.2 atomicwrites==1.3.0 attrs==19.1.0 -babel==2.6.0 +babel==2.7.0 certifi==2019.3.9 chardet==3.0.4 -commonmark==0.8.1 +commonmark==0.9.0 +configparser==3.7.4 ; python_version < '3.2' +contextlib2==0.5.5 ; python_version < '3.5' docutils==0.14 funcsigs==1.0.2 ; python_version < '3.3' future==0.17.1 gunicorn==19.9.0 idna==2.8 imagesize==1.1.0 +importlib-metadata==0.17 jinja2==2.10.1 lxml==4.3.3 markdown==2.6.11 markupsafe==1.1.1 mirakuru==1.1.0 -mock==2.0.0 +mock==3.0.5 more-itertools==5.0.0 nose==1.3.7 nosehtml==0.4.5 packaging==19.0 pathlib2==2.3.2 ; python_version < '3' pathtools==0.1.2 -pbr==5.1.3 -pluggy==0.9.0 +pluggy==0.12.0 port-for==0.4 -psutil==5.6.1 +psutil==5.6.2 py==1.8.0 pygithub3==0.5.1 ; python_version < '3' -pygments==2.3.1 +pygments==2.4.2 pyparsing==2.4.0 pytest-html==1.20.0 pytest-metadata==1.8.0 -pytest-postgresql==1.4.0 +pytest-postgresql==1.4.1 pytest-pythonpath==0.7.3 -pytest==4.4.1 +pytest==4.6.2 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.1 recommonmark==0.5.0 -requests==2.21.0 +requests==2.22.0 scandir==1.10.0 ; python_version < '3.5' selenium==3.141.0 six==1.11.0 @@ -50,9 +52,11 @@ snowballstemmer==1.2.1 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.3 sphinx==1.8.5 -sphinxcontrib-websupport==1.1.0 -testfixtures==6.7.0 +sphinxcontrib-websupport==1.1.2 +testfixtures==6.8.2 twill==0.9.1 ; python_version < '3' typing==3.6.6 ; python_version < '3.5' -urllib3==1.24.2 ; python_version == '2.7' +urllib3==1.25.3 ; python_version == '2.7' watchdog==0.9.0 +wcwidth==0.1.7 ; sys_platform != 'win32' +zipp==0.5.1 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 56b6f3858f6..17fe8f5fa7d 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -1,9 +1,10 @@ -i https://wheels.galaxyproject.org/simple --extra-index-url https://pypi.python.org/simple adal==1.2.1 -amqp==2.4.2 +amqp==2.5.0 appdirs==1.4.3 asn1crypto==0.24.0 +attrs==19.1.0 avro==1.8.1 ; python_version < '3' azure-common==1.1.14 azure-cosmosdb-nspkg==2.0.2 @@ -16,9 +17,9 @@ azure-mgmt-resource==2.0.0 azure-mgmt-storage==2.0.0 azure-nspkg==3.0.2 azure-storage-blob==1.3.1 -azure-storage-common==1.4.0 +azure-storage-common==1.4.2 azure-storage-nspkg==3.1.0 -babel==2.6.0 +babel==2.7.0 bagit==1.6.4 bcrypt==3.1.6 bdbag==1.4.1 @@ -28,21 +29,21 @@ bleach==3.1.0 boltons==19.1.0 boto3==1.9.114 boto==2.49.0 -botocore==1.12.133 +botocore==1.12.164 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 -cachetools==3.1.0 +cachetools==3.1.1 certifi==2019.3.9 cffi==1.12.3 chardet==3.0.4 -cheetah3==3.2.2 +cheetah3==3.2.3 cliff==2.14.1 -cloudauthz==0.3.0 +cloudauthz==0.2.0 cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' -cryptography==2.6.1 +cryptography==2.7 cwltool==1.0.20180721142728 debtcollector==1.21.0 decorator==4.4.0 @@ -65,7 +66,7 @@ google-auth-httplib2==0.0.3 google-auth==1.6.3 gxformat2==0.8.3 h5py==2.9.0 -httplib2==0.12.1 +httplib2==0.13.0 idna==2.8 ipaddress==1.0.22 ; python_version < '3.3' isa-rwval==0.10.7 @@ -74,12 +75,12 @@ isodate==0.6.0 jmespath==0.9.4 jsonpatch==1.23 jsonpointer==2.0 -jsonschema==2.6.0 +jsonschema==3.0.1 keystoneauth1==3.14.0 -kombu==4.5.0 +kombu==4.6.1 lockfile==0.12.2 lxml==4.3.3 -mako==1.0.9 +mako==1.0.12 markupsafe==1.1.1 mercurial==3.7.3 ; python_version < '3' mistune==0.8.4 @@ -94,35 +95,35 @@ netifaces==0.10.9 networkx==1.11 nodeenv==1.3.3 nose==1.3.7 -numpy==1.16.2 +numpy==1.16.4 oauth2client==4.1.3 oauthlib==3.0.1 openstacksdk==0.17.0 os-client-config==1.32.0 -os-service-types==1.6.0 +os-service-types==1.7.0 osc-lib==1.12.1 -oslo.config==6.8.1 +oslo.config==6.9.0 oslo.context==2.22.1 oslo.i18n==3.23.1 -oslo.log==3.42.3 -oslo.serialization==2.28.2 -oslo.utils==3.40.3 +oslo.log==3.44.0 +oslo.serialization==2.29.1 +oslo.utils==3.41.0 packaging==19.0 paramiko==2.4.2 parsley==1.3 -paste==3.0.8 +paste==3.0.8 ; python_version == '2.7' pastedeploy==2.0.1 pastescript==3.1.0 pathlib2==2.3.2 ; python_version < '3' -pbr==5.1.3 +pbr==5.2.1 prettytable==0.7.2 prov==1.5.1 -psutil==5.6.1 -pulsar-galaxy-lib==0.11.0 -pyasn1-modules==0.2.4 +psutil==5.6.2 +pulsar-galaxy-lib==0.12.1 +pyasn1-modules==0.2.5 pyasn1==0.4.5 pycparser==2.19 -pycryptodome==3.8.1 +pycryptodome==3.8.2 pyeventsystem==0.1.0 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' pyjwt==1.7.1 @@ -131,6 +132,7 @@ pynacl==1.3.0 pyopenssl==19.0.0 pyparsing==2.4.0 pyperclip==1.7.0 +pyrsistent==0.15.2 pysam==0.15.2 pysftp==0.2.9 python-cinderclient==4.0.0 @@ -144,32 +146,32 @@ python-novaclient==11.0.0 python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.1 rdflib-jsonld==0.4.0 rdflib==4.2.2 repoze.lru==0.7 requests-oauthlib==1.2.0 requests-toolbelt==0.9.1 -requests==2.21.0 +requests==2.22.0 requestsexceptions==1.4.0 -rfc3986==1.2.0 +rfc3986==1.3.2 routes==2.4.1 rsa==4.0 ruamel.ordereddict==0.4.13 ; platform_python_implementation == 'CPython' and python_version <= '2.7' -ruamel.yaml==0.15.92 -s3transfer==0.2.0 +ruamel.yaml==0.15.97 +s3transfer==0.2.1 scandir==1.10.0 ; python_version < '3.5' schema-salad==2.7.20181126142424 shellescape==3.4.1 simplejson==3.16.0 six==1.11.0 -social-auth-core[openidconnect]==3.1.0+gx0 +social-auth-core[openidconnect]==3.1.0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.18 sqlparse==0.3.0 stevedore==1.30.1 -subprocess32==3.5.3 ; python_version < '3.0' +subprocess32==3.5.4 ; python_version < '3.0' svgwrite==1.2.1 tempita==0.5.2 tenacity==4.12.0 @@ -178,10 +180,10 @@ typing==3.6.6 ; python_version < '3.5' tzlocal==1.5.1 unicodecsv==0.14.1 ; python_version < '3.0' uritemplate==3.0.0 -urllib3==1.24.2 ; python_version == '2.7' +urllib3==1.25.3 ; python_version == '2.7' uwsgi==2.0.18 vine==1.3.0 -warlock==1.3.0 +warlock==1.3.3 wcwidth==0.1.7 ; sys_platform != 'win32' webencodings==0.5.1 webob==1.8.5 From 35629c33db9f37ea1f2448885e8cf1a2d7b635fc Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 9 Jun 2019 09:46:10 -0700 Subject: [PATCH 45/88] Revert social-auth-core version change. --- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 17fe8f5fa7d..a62c23d7f0c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -165,7 +165,7 @@ schema-salad==2.7.20181126142424 shellescape==3.4.1 simplejson==3.16.0 six==1.11.0 -social-auth-core[openidconnect]==3.1.0 +social-auth-core[openidconnect]==3.1.0+gx0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.33.11 sqlalchemy==1.2.18 From fed80f19f6340f01401390c1ef5850e1fd71153a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 9 Jun 2019 19:00:31 +0100 Subject: [PATCH 46/88] Pin SQLAlchemy and update social_auth_core pin - SQLAlchemy 1.3 introduced changes that break some queries. - Commit 5723e15cce8bc493a782b5aa18b894d230da6261 did not update the pin on `social_auth_core` in the Pipfile. - Remove spurious `; python_version == '2.7'` produced by pipenv for `paste`. - Run `make update-dependencies`. --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 4 ++-- .../dependencies/pipfiles/default/pinned-requirements.txt | 6 +++--- lib/galaxy/dependencies/pipfiles/update.sh | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 6741c925615..5d10c76a64a 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -32,7 +32,7 @@ numpy = "*" bx-python = "*" MarkupSafe = "*" PyYAML = "*" -SQLAlchemy = "*" +SQLAlchemy = "<1.3" SQLAlchemy-Utils = "*" Mercurial = {version = "<=3.7.3", markers = "python_version < '3'"} nodeenv = "*" @@ -75,7 +75,7 @@ pyparsing = "*" "Fabric3" = "*" paramiko = "*" python-genomespaceclient = "<2.0" -social_auth_core = {version = "==3.1.0", extras = ['openidconnect']} +social_auth_core = {version = "==3.1.0+gx0", extras = ['openidconnect']} cloudauthz = "<=0.2.0" gxformat2 = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index a62c23d7f0c..cc958eb70b4 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -111,7 +111,7 @@ oslo.utils==3.41.0 packaging==19.0 paramiko==2.4.2 parsley==1.3 -paste==3.0.8 ; python_version == '2.7' +paste==3.0.8 pastedeploy==2.0.1 pastescript==3.1.0 pathlib2==2.3.2 ; python_version < '3' @@ -167,8 +167,8 @@ simplejson==3.16.0 six==1.11.0 social-auth-core[openidconnect]==3.1.0+gx0 sqlalchemy-migrate==0.12.0 -sqlalchemy-utils==0.33.11 -sqlalchemy==1.2.18 +sqlalchemy-utils==0.34.0 +sqlalchemy==1.2.19 sqlparse==0.3.0 stevedore==1.30.1 subprocess32==3.5.4 ; python_version < '3.0' diff --git a/lib/galaxy/dependencies/pipfiles/update.sh b/lib/galaxy/dependencies/pipfiles/update.sh index ff67d81805c..edafb36cb7e 100755 --- a/lib/galaxy/dependencies/pipfiles/update.sh +++ b/lib/galaxy/dependencies/pipfiles/update.sh @@ -65,6 +65,7 @@ for env in $ENVS; do -e "s/^futures==\([^ ;]\{1,\}\).*$/futures==\1 ; python_version == '2.6' or python_version == '2.7'/" \ -e "s/^monotonic==\([^ ;]\{1,\}\).*$/monotonic==\1/" \ -e "s/^more-itertools==\([^ ;]\{1,\}\).*$/more-itertools==\1/" \ + -e "s/^paste==\([^ ;]\{1,\}\).*$/paste==\1/" \ -e "s/^py2-ipaddress==\([^ ;]\{1,\}\).*$/py2-ipaddress==\1 ; python_version < '3'/" \ -e "s/^pyinotify==\([^ ;]\{1,\}\).*$/pyinotify==\1 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5'/" \ -e "s/^python-dateutil==\([^ ;]\{1,\}\).*$/python-dateutil==\1/" \ From d787a2827a337977b75ee58fc11416f52c3eaddd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 10 Jun 2019 09:19:20 -0400 Subject: [PATCH 47/88] Update lib/galaxy/util/plugin_config.py Co-Authored-By: Marius van den Beek --- lib/galaxy/util/plugin_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 71fb410aab0..894601e7f75 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -30,7 +30,9 @@ def plugins_dict(module, plugin_type_identifier): return plugin_dict -def load_plugins(plugins_dict, plugin_source, extra_kwds={}, plugin_type_keys=['type']): +def load_plugins(plugins_dict, plugin_source, extra_kwds=None, plugin_type_keys=('type',)): + if extra_kwds is None: + extra_kwds = {} source_type, source = plugin_source if source_type == "xml": return __load_plugins_from_element(plugins_dict, source, extra_kwds) From 5651edcf2499e0e7ffaa1bc4d7300509ac94d4a8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 10 Jun 2019 09:19:40 -0400 Subject: [PATCH 48/88] Update lib/galaxy/webapps/galaxy/config_schema.yml Co-Authored-By: Marius van den Beek --- lib/galaxy/webapps/galaxy/config_schema.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index bd393bec68a..91875eff914 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2369,6 +2369,7 @@ mapping: desc: | Rather than specifying a dependency_resolvers_config_file, the definition of the resolvers to enable can be embedded into Galaxy's config with this option. + This has no effect if a dependency_resolvers_config_file is used. default_job_resubmission_condition: type: str From 9043dd18c6c0b7766ce4e606faf503330b1de306 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Mon, 10 Jun 2019 10:31:59 -0400 Subject: [PATCH 49/88] [19.05] Add recent merges to release notes --- doc/source/releases/19.05.rst | 5 +++++ doc/source/releases/19.05_prs.rst | 2 ++ 2 files changed, 7 insertions(+) diff --git a/doc/source/releases/19.05.rst b/doc/source/releases/19.05.rst index 7166c497f17..160ab07a718 100644 --- a/doc/source/releases/19.05.rst +++ b/doc/source/releases/19.05.rst @@ -393,6 +393,11 @@ Enhancements Fixes ------------------------------- +* Explicitly specify system user under which Kubernetes jobs should run. + `Pull Request 8103`_ +* Ensure Kubernetes job querries filter the same namespace as defined in the + job destination. + `Pull Request 8092`_ * Fix client styleguide asset handling. `Pull Request 7664`_ * Handle subworkflows in view workflow mako diff --git a/doc/source/releases/19.05_prs.rst b/doc/source/releases/19.05_prs.rst index 59ca8a5b131..4498056e1a8 100644 --- a/doc/source/releases/19.05_prs.rst +++ b/doc/source/releases/19.05_prs.rst @@ -1,5 +1,7 @@ .. github_links +.. _Pull Request 8103: https://github.com/galaxyproject/galaxy/pull/8103 +.. _Pull Request 8092: https://github.com/galaxyproject/galaxy/pull/8092 .. _Pull Request 4659: https://github.com/galaxyproject/galaxy/pull/4659 .. _Pull Request 5495: https://github.com/galaxyproject/galaxy/pull/5495 .. _Pull Request 6321: https://github.com/galaxyproject/galaxy/pull/6321 From 15239f1674081ab51ab8dd75a9a40cf1bfaa93e8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 10 Jun 2019 15:01:46 -0400 Subject: [PATCH 50/88] Update version to 19.05 --- lib/galaxy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/version.py b/lib/galaxy/version.py index 701b600c93d..c5c6eba444f 100644 --- a/lib/galaxy/version.py +++ b/lib/galaxy/version.py @@ -1,3 +1,3 @@ VERSION_MAJOR = "19.05" -VERSION_MINOR = "dev" +VERSION_MINOR = None VERSION = VERSION_MAJOR + ('.' + VERSION_MINOR if VERSION_MINOR else '') From 1ce7b2a1af72d277001f9c3e64dd3f1188ae3b5d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 7 Jun 2019 10:01:11 -0400 Subject: [PATCH 51/88] Fix Pulsar job files API to allow transferring directory content. --- lib/galaxy/webapps/galaxy/api/job_files.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/job_files.py b/lib/galaxy/webapps/galaxy/api/job_files.py index 813db8bca9c..5f571b94049 100644 --- a/lib/galaxy/webapps/galaxy/api/job_files.py +++ b/lib/galaxy/webapps/galaxy/api/job_files.py @@ -99,6 +99,8 @@ class JobFilesAPIController(BaseAPIController): else: input_file = payload.get("file", payload.get("__file", None)).file + target_dir = os.path.dirname(path) + util.safe_makedirs(target_dir) try: shutil.move(input_file.name, path) finally: From c76bbffe2d103752832d0272e3aaeb2100152703 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jun 2019 00:45:08 +0100 Subject: [PATCH 52/88] Update Python dependencies, unpin SQLAlchemy --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../pipfiles/default/pinned-dev-requirements.txt | 4 ++-- .../dependencies/pipfiles/default/pinned-requirements.txt | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 5d10c76a64a..02777c6b6b6 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -32,7 +32,7 @@ numpy = "*" bx-python = "*" MarkupSafe = "*" PyYAML = "*" -SQLAlchemy = "<1.3" +SQLAlchemy = "*" SQLAlchemy-Utils = "*" Mercurial = {version = "<=3.7.3", markers = "python_version < '3'"} nodeenv = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index d87cc2cc5ba..c95d3ff3274 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -18,7 +18,7 @@ idna==2.8 imagesize==1.1.0 importlib-metadata==0.17 jinja2==2.10.1 -lxml==4.3.3 +lxml==4.3.4 markdown==2.6.11 markupsafe==1.1.1 mirakuru==1.1.0 @@ -53,7 +53,7 @@ sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.3 sphinx==1.8.5 sphinxcontrib-websupport==1.1.2 -testfixtures==6.8.2 +testfixtures==6.9.0 twill==0.9.1 ; python_version < '3' typing==3.6.6 ; python_version < '3.5' urllib3==1.25.3 ; python_version == '2.7' diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index cc958eb70b4..a8afc0745a0 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -29,7 +29,7 @@ bleach==3.1.0 boltons==19.1.0 boto3==1.9.114 boto==2.49.0 -botocore==1.12.164 +botocore==1.12.165 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 @@ -79,7 +79,7 @@ jsonschema==3.0.1 keystoneauth1==3.14.0 kombu==4.6.1 lockfile==0.12.2 -lxml==4.3.3 +lxml==4.3.4 mako==1.0.12 markupsafe==1.1.1 mercurial==3.7.3 ; python_version < '3' @@ -109,7 +109,7 @@ oslo.log==3.44.0 oslo.serialization==2.29.1 oslo.utils==3.41.0 packaging==19.0 -paramiko==2.4.2 +paramiko==2.5.0 parsley==1.3 paste==3.0.8 pastedeploy==2.0.1 @@ -168,7 +168,7 @@ six==1.11.0 social-auth-core[openidconnect]==3.1.0+gx0 sqlalchemy-migrate==0.12.0 sqlalchemy-utils==0.34.0 -sqlalchemy==1.2.19 +sqlalchemy==1.3.4 sqlparse==0.3.0 stevedore==1.30.1 subprocess32==3.5.4 ; python_version < '3.0' From 1fd248311fec956e4cd9fef6bb5bf7e9b84a2b83 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jun 2019 11:31:35 +0100 Subject: [PATCH 53/88] Fix traceback in ``printquery()`` Fix the following traceback: ``` /usr/users/ga002/soranzon/software/galaxyproject_galaxy/scripts/db_shell.py in printquery(statement, bind) 60 if bind is None: 61 bind = statement.session.get_bind( ---> 62 statement._mapper_zero_or_none()) 63 statement = statement.statement 64 elif bind is None: AttributeError: 'Query' object has no attribute '_mapper_zero_or_none' ``` due to the removal of `_mapper_zero_or_none()` in SQLAlchemy commit https://github.com/sqlalchemy/sqlalchemy/commit/234a5b9723fbbb9747c0f9e3917baf8500b73370 --- scripts/db_shell.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/db_shell.py b/scripts/db_shell.py index 2ec27eb2db4..ddf85059796 100644 --- a/scripts/db_shell.py +++ b/scripts/db_shell.py @@ -58,8 +58,7 @@ def printquery(statement, bind=None): import sqlalchemy.orm if isinstance(statement, sqlalchemy.orm.Query): if bind is None: - bind = statement.session.get_bind( - statement._mapper_zero_or_none()) + bind = statement.session.get_bind() statement = statement.statement elif bind is None: bind = statement.bind From 36a2412fb7ddbf791178ff83e01c405849b669b9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jun 2019 11:34:04 +0100 Subject: [PATCH 54/88] Fix ``join()`` call for SQLAlchemy 1.3 Fix the following traceback: ``` galaxy.jobs.handler ERROR 2019-06-11 01:33:33,567 Exception in monitor_step Traceback (most recent call last): File "/galaxy/lib/galaxy/jobs/handler.py", line 234, in __monitor self.__monitor_step() File "/galaxy/lib/galaxy/jobs/handler.py", line 248, in __monitor_step self.__handle_waiting_jobs() File "/galaxy/lib/galaxy/jobs/handler.py", line 318, in __handle_waiting_jobs jobs_to_check = self.__filter_jobs_with_invalid_input_states(jobs_to_check) File "/galaxy/lib/galaxy/jobs/handler.py", line 437, in __filter_jobs_with_invalid_input_states ).join(job_to_input) \ File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2239, in join from_joinpoint=from_joinpoint, File "", line 2, in _join File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/base.py", line 220, in generate fn(self, *args[1:], **kw) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2418, in _join left, right, onclause, prop, create_aliases, outerjoin, full File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2441, in _join_left_to_right ) = self._join_determine_implicit_left_side(left, right, onclause) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2572, in _join_determine_implicit_left_side "Can't determine which FROM clause to join " InvalidRequestError: Can't determine which FROM clause to join from, there are multiple FROMS which can join to this entity. Try adding an explicit ON clause to help resolve the ambiguity. ``` --- lib/galaxy/jobs/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 33a705caccc..f2202d4d6f4 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -434,7 +434,7 @@ class JobHandlerQueue(Monitors): model.Dataset.deleted, model.Dataset.purged, model.Dataset.state, - ).join(job_to_input) \ + ).join(job_to_input.job) \ .join(input_association) \ .join(model.Dataset) \ .filter(model.Job.id.in_(job_ids_to_check)) \ From 18ed4be13c989321c04b634d301fd2a014af1f6d Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 5 Jun 2019 15:53:05 -0400 Subject: [PATCH 55/88] add 19.05 RN to index --- doc/source/releases/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/releases/index.rst b/doc/source/releases/index.rst index 1a59eb20ee8..edc8647c307 100644 --- a/doc/source/releases/index.rst +++ b/doc/source/releases/index.rst @@ -4,6 +4,7 @@ Releases .. toctree:: :maxdepth: 1 + 19.05_announce 19.01_announce 18.09_announce 18.05_announce From 1c6447359ac05600ed9a8f4d3d39836103010f07 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jun 2019 12:44:56 +0100 Subject: [PATCH 56/88] Fix string passed to ``Query.filter()``, not allowed in SQLAlchemy 1.3 Fix traceback: ``` File "/galaxy/lib/galaxy/webapps/tool_shed/controllers/repository.py", line 1867, in manage_repository avg_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, repository, webapp_model=trans.model) File "/galaxy/lib/galaxy/model/item_attrs.py", line 29, in get_ave_item_rating_data ave_rating = db_session.query(func.avg(item_rating_assoc_class.rating)).filter(item_id_filter).scalar() File "", line 2, in filter File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/base.py", line 220, in generate fn(self, *args[1:], **kw) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 1760, in filter criterion = expression._expression_literal_as_text(criterion) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4445, in _expression_literal_as_text return _literal_as_text(element) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4468, in _literal_as_text return _literal_as(element, _no_text_coercion) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4454, in _literal_as return text_fallback(element) File "/galaxy_venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4498, in _no_text_coercion "extra": "%s " % extra if extra else "", ArgumentError: Textual SQL expression u'repository_id=1' should be explicitly declared as text(u'repository_id=1') ``` --- lib/galaxy/model/item_attrs.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/item_attrs.py b/lib/galaxy/model/item_attrs.py index bf32ed87a5a..d054c8bbc42 100644 --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -74,13 +74,10 @@ class UsesItemRatings(object): item_rating_assoc_class = '%sRatingAssociation' % item.__class__.__name__ return getattr(webapp_model, item_rating_assoc_class, None) - def _get_item_id_filter_str(self, item, item_rating_assoc_class, webapp_model=None): + def _get_item_id_filter_str(self, item, item_rating_assoc_class): # Get foreign key in item-rating association table that references item table. - if webapp_model is None: - webapp_model = galaxy.model item_fk = get_foreign_key(item_rating_assoc_class, item) - # TODO: can we provide a better filter than a raw string? - return "%s=%i" % (item_fk.parent.name, item.id) + return item_fk.parent == item.id class UsesAnnotations(object): @@ -122,7 +119,6 @@ def get_item_annotation_obj(db_session, user, item): # Get annotation association object. annotation_assoc = db_session.query(annotation_assoc_class).filter_by(user=user) - # TODO: use filtering like that in _get_item_id_filter_str() if item.__class__ == galaxy.model.History: annotation_assoc = annotation_assoc.filter_by(history=item) elif item.__class__ == galaxy.model.HistoryDatasetAssociation: From 2e2e8de0fd4e4373492e59af881f8cc7468274f9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jun 2019 16:46:28 +0100 Subject: [PATCH 57/88] Fix string passed to ``Query.filter()``, not allowed in SQLAlchemy 1.3 Fix traceback: ``` File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/webapps/tool_shed/controllers/repository.py", line 1095, in find_tools return self.matched_repository_grid(trans, **kwd) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/webapps/reports/framework/grids.py", line 73, in __call__ query = self.build_initial_query(trans, **kwargs) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/tool_shed/grids/repository_grids.py", line 376, in build_initial_query .filter(or_(*clause_list)) \ File "", line 2, in or_ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 2058, in or_ return cls._construct(operators.or_, False_, True_, *clauses) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 1958, in _construct for clause in util.coerce_generator_arg(clauses) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4445, in _expression_literal_as_text return _literal_as_text(element) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4468, in _literal_as_text return _literal_as(element, _no_text_coercion) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4454, in _literal_as return text_fallback(element) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 4498, in _no_text_coercion "extra": "%s " % extra if extra else "", ArgumentError: Textual SQL expression u'repository_metadata.repos...' should be explicitly declared as text(u'repository_metadata.repos...') ``` --- lib/tool_shed/grids/repository_grids.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index be1b51994e8..ba664184fab 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -364,10 +364,9 @@ class MatchedRepositoryGrid(grids.Grid): if match_tuples: for match_tuple in match_tuples: repository_id, changeset_revision = match_tuple - clause_list.append("%s=%d and %s='%s'" % (model.RepositoryMetadata.table.c.repository_id, - int(repository_id), - model.RepositoryMetadata.table.c.changeset_revision, - changeset_revision)) + clause_list.append(and_( + model.RepositoryMetadata.repository_id == int(repository_id), + model.RepositoryMetadata.changeset_revision == changeset_revision)) return trans.sa_session.query(model.RepositoryMetadata) \ .join(model.Repository) \ .filter(and_(model.Repository.table.c.deleted == false(), From 482eb71525c544d5eb30bd1d5623328288e19688 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 11 Jun 2019 12:20:14 -0400 Subject: [PATCH 58/88] improve contrastr of popovers affects tours --- client/galaxy/style/scss/ui.scss | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/galaxy/style/scss/ui.scss b/client/galaxy/style/scss/ui.scss index 85f3f6d025b..e0fc091a557 100644 --- a/client/galaxy/style/scss/ui.scss +++ b/client/galaxy/style/scss/ui.scss @@ -256,6 +256,8 @@ $ui-margin-horizontal-large: $margin-v * 2; .popover { max-width: 700px; .popover-header { + color: $brand-white; + background-color: $brand-primary; i { @extend .float-right; @extend .pl-4; From b0761c3f7bd9e9717ed7bfa0229a0dc2edd5a3ee Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 11 Jun 2019 12:37:42 -0400 Subject: [PATCH 59/88] improve contrast of links in all bs alerts --- client/galaxy/style/scss/overrides.scss | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/galaxy/style/scss/overrides.scss b/client/galaxy/style/scss/overrides.scss index d823ca46904..6b922f274ed 100644 --- a/client/galaxy/style/scss/overrides.scss +++ b/client/galaxy/style/scss/overrides.scss @@ -85,3 +85,9 @@ pre.code { color: white; padding: 1em; } + +.alert-info, .alert-danger, .alert-warning, .alert-success { + a { + font-weight: bold; + } +} From c9a262da36ba2323bbeb7740bb773c488a5164b7 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 11 Jun 2019 12:48:35 -0400 Subject: [PATCH 60/88] increase visibility iof dropdown section headers --- client/galaxy/style/scss/overrides.scss | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/galaxy/style/scss/overrides.scss b/client/galaxy/style/scss/overrides.scss index 6b922f274ed..f1c8f3f2b1b 100644 --- a/client/galaxy/style/scss/overrides.scss +++ b/client/galaxy/style/scss/overrides.scss @@ -91,3 +91,10 @@ pre.code { font-weight: bold; } } + +.dropdown-header { + font-size: 1rem; + color: $brand-dark; + padding: .5rem 1.5rem .5rem 1rem; + font-weight: bold; +} From 102b2d90b213aec7845f3debd26f8fab0edb4f32 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Mar 2019 10:31:34 -0400 Subject: [PATCH 61/88] Fix SQLAlchemy deprecation warning Fix the following: ``` SADeprecationWarning: The select.for_update parameter is deprecated and will be removed in a future release. Please refer to the Select.with_for_update() to specify the structure of the ``FOR UPDATE`` clause. ``` --- lib/galaxy/model/mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 25f91256140..e9858e23292 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -2728,7 +2728,7 @@ def db_next_hid(self, n=1): trans = session.begin() try: if "postgres" not in session.bind.dialect.name: - next_hid = select([table.c.hid_counter], table.c.id == model.cached_id(self), for_update=True).scalar() + next_hid = select([table.c.hid_counter], table.c.id == model.cached_id(self)).with_for_update().scalar() table.update(table.c.id == self.id).execute(hid_counter=(next_hid + n)) else: stmt = table.update().where(table.c.id == model.cached_id(self)).values(hid_counter=(table.c.hid_counter + n)).returning(table.c.hid_counter) From 69e29be871a283e9839c9b877a5fec5af1870495 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 11 Jun 2019 12:55:07 -0400 Subject: [PATCH 62/88] add docs for new overrides --- client/galaxy/style/scss/overrides.scss | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/galaxy/style/scss/overrides.scss b/client/galaxy/style/scss/overrides.scss index f1c8f3f2b1b..456f65fc816 100644 --- a/client/galaxy/style/scss/overrides.scss +++ b/client/galaxy/style/scss/overrides.scss @@ -86,12 +86,14 @@ pre.code { padding: 1em; } +// increase visibility of links within alert boxes .alert-info, .alert-danger, .alert-warning, .alert-success { a { font-weight: bold; } } +// increase visibility of dropdown menu section headers .dropdown-header { font-size: 1rem; color: $brand-dark; From c325ff72fff304714dc9eaabe1ab9a5820b920fa Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jun 2019 22:04:21 +0100 Subject: [PATCH 63/88] Do not ``del`` ORM object attribute In SQLAlchemy 1.3 this is now equivalent to an SQL "= None", see https://docs.sqlalchemy.org/en/13/changelog/migration_13.html#del-implemented-for-orm-attributes Fix the following traceback: ``` galaxy.web.framework.decorators ERROR 2019-06-11 16:12:17,022 Uncaught exception in exposed API method: Traceback (most recent call last): File "/galaxy/lib/galaxy/web/framework/decorators.py", line 282, in decorator rval = func(self, trans, *args, **kwargs) File "/galaxy/lib/galaxy/webapps/galaxy/api/history_contents.py", line 176, in show return self.__show_dataset(trans, id, **kwd) File "/galaxy/lib/galaxy/webapps/galaxy/api/history_contents.py", line 263, in __show_dataset **self._parse_serialization_params(kwd, 'detailed')) File "/galaxy/lib/galaxy/managers/base.py", line 692, in serialize_to_view return self.serialize(item, all_keys, **context) File "/galaxy/lib/galaxy/managers/hdas.py", line 369, in serialize return super(HDASerializer, self).serialize(hda, keys, user=user, **context) File "/galaxy/lib/galaxy/managers/datasets.py", line 604, in serialize serialized = super(DatasetAssociationSerializer, self).serialize(dataset_assoc, keys, **context) File "/galaxy/lib/galaxy/managers/base.py", line 605, in serialize returned[key] = self.serializers[key](item, key, **context) File "/galaxy/lib/galaxy/managers/hdas.py", line 352, in history_id=self.app.security.encode_id(i.history.id), AttributeError: 'NoneType' object has no attribute 'id' ``` --- lib/galaxy/model/__init__.py | 5 +---- lib/galaxy/tools/actions/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index bb8db8a0292..7d0f91185dd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1634,13 +1634,10 @@ class History(HasTags, Dictifiable, UsesAnnotations, HasName, RepresentById): set_genome = genome_build not in [None, '?'] for i, dataset in enumerate(datasets): dataset.hid = base_hid + i - # Don't let SA manage this. - delattr(dataset, "history") + dataset.history = self dataset.history_id = cached_id(self) if set_genome: self.genome_build = genome_build - for dataset in datasets: - dataset.history_id = cached_id(self) return datasets def add_dataset_collection(self, history_dataset_collection, set_hid=True): diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 3125f399355..fe7d3efa002 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -518,9 +518,8 @@ class DefaultToolAction(object): add_datasets_timer = ExecutionTimer() # Add all the top-level (non-child) datasets to the history unless otherwise specified datasets_to_persist = [] - for name in out_data.keys(): + for name, data in out_data.items(): if name not in child_dataset_names and name not in incoming: # don't add children; or already existing datasets, i.e. async created - data = out_data[name] datasets_to_persist.append(data) # Set HID and add to history. # This is brand new and certainly empty so don't worry about quota. From d4f15f28bf8d6e8096d0d7e72ea0a3b311d6bb27 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Jun 2019 11:44:32 +0200 Subject: [PATCH 64/88] Ensure DEFAULT_TOOL_TEST_WAIT is integer This fixes ``` Test 'toolshed.g2.bx.psu.edu/repos/iuc/mothur_unifrac_weighted/mothur_unifrac_weighted/1.39.5.0-0' failed Traceback (most recent call last): File "/var/jenkins/shiningpanda/jobs/35ce1600/virtualenvs/d41d8cd9/lib/python3.5/site-packages/ephemeris/shed_tools.py", line 338, in run_test register_job_data=register, quiet=True, test_history=test_history, File "/var/jenkins/shiningpanda/jobs/35ce1600/virtualenvs/d41d8cd9/lib/python3.5/site-packages/galaxy/tools/verify/interactor.py", line 733, in verify_tool stage_data_in_history(galaxy_interactor, tool_id, testdef.test_data(), history=test_history, force_path_paste=force_path_paste) File "/var/jenkins/shiningpanda/jobs/35ce1600/virtualenvs/d41d8cd9/lib/python3.5/site-packages/galaxy/tools/verify/interactor.py", line 86, in stage_data_in_history upload_wait() File "/var/jenkins/shiningpanda/jobs/35ce1600/virtualenvs/d41d8cd9/lib/python3.5/site-packages/galaxy/tools/verify/interactor.py", line 369, in return lambda: self.wait_for_job(jobs[0]["id"], history_id, DEFAULT_TOOL_TEST_WAIT) File "/var/jenkins/shiningpanda/jobs/35ce1600/virtualenvs/d41d8cd9/lib/python3.5/site-packages/galaxy/tools/verify/interactor.py", line 238, in wait_for_job self.wait_for(lambda: not self.__job_ready(job_id, history_id), maxseconds=maxseconds) File "/var/jenkins/shiningpanda/jobs/35ce1600/virtualenvs/d41d8cd9/lib/python3.5/site-packages/galaxy/tools/verify/interactor.py", line 245, in wait_for while slept <= walltime_exceeded: TypeError: unorderable types: int() <= str() ``` that we encountered when we ran tool tests using ephemeris against python 3. --- lib/galaxy/tools/verify/interactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 0898b7c7c39..f556706da34 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -42,7 +42,7 @@ log = getLogger(__name__) VERBOSE_ERRORS = util.asbool(os.environ.get("GALAXY_TEST_VERBOSE_ERRORS", False)) UPLOAD_ASYNC = util.asbool(os.environ.get("GALAXY_TEST_UPLOAD_ASYNC", True)) ERROR_MESSAGE_DATASET_SEP = "--------------------------------------" -DEFAULT_TOOL_TEST_WAIT = os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400) +DEFAULT_TOOL_TEST_WAIT = int(os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400)) DEFAULT_FTYPE = 'auto' # This following default dbkey was traditionally hg17 before Galaxy 18.05, From a81c8102727c8adce681ce4a8be3aa03aa039850 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 12 Jun 2019 11:09:37 +0100 Subject: [PATCH 65/88] Small code simplification --- lib/galaxy/tools/verify/interactor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index f556706da34..2504b0dce56 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -48,7 +48,6 @@ DEFAULT_FTYPE = 'auto' # This following default dbkey was traditionally hg17 before Galaxy 18.05, # restore this behavior by setting GALAXY_TEST_DEFAULT_DBKEY to hg17. DEFAULT_DBKEY = os.environ.get("GALAXY_TEST_DEFAULT_DBKEY", "?") -DEFAULT_MAX_SECS = DEFAULT_TOOL_TEST_WAIT class OutputsDict(OrderedDict): @@ -1027,7 +1026,7 @@ class ToolTestDescription(object): def __init__(self, processed_test_dict): test_index = processed_test_dict["test_index"] name = processed_test_dict.get('name', 'Test-%d' % (test_index + 1)) - maxseconds = processed_test_dict.get('maxseconds', DEFAULT_MAX_SECS) + maxseconds = processed_test_dict.get('maxseconds', DEFAULT_TOOL_TEST_WAIT) if maxseconds is not None: maxseconds = int(maxseconds) From 83b72c1f17ac59056f2022d8d28a4477beddd651 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Jun 2019 12:09:46 +0200 Subject: [PATCH 66/88] Allow specifying maxseconds in verify_tool --- lib/galaxy/tools/verify/interactor.py | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 2504b0dce56..339ac8cc7cc 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -72,7 +72,7 @@ class OutputsDict(OrderedDict): return item -def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history=None, force_path_paste=False): +def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history=None, force_path_paste=False, maxseconds=DEFAULT_TOOL_TEST_WAIT): # Upload any needed files upload_waits = [] @@ -80,12 +80,20 @@ def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history=Non if UPLOAD_ASYNC: for test_data in all_test_data: - upload_waits.append(galaxy_interactor.stage_data_async(test_data, history, tool_id, force_path_paste=force_path_paste)) + upload_waits.append(galaxy_interactor.stage_data_async(test_data, + history, + tool_id, + force_path_paste=force_path_paste, + maxseconds=maxseconds)) for upload_wait in upload_waits: upload_wait() else: for test_data in all_test_data: - upload_wait = galaxy_interactor.stage_data_async(test_data, history, tool_id, force_path_paste=force_path_paste) + upload_wait = galaxy_interactor.stage_data_async(test_data, + history, + tool_id, + force_path_paste=force_path_paste, + maxseconds=maxseconds) upload_wait() @@ -239,7 +247,9 @@ class GalaxyInteractorApi(object): def wait_for(self, func, **kwd): sleep_amount = 0.2 slept = 0 - walltime_exceeded = kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT) + walltime_exceeded = kwd.get("maxseconds") + if walltime_exceeded is None: + walltime_exceeded = DEFAULT_TOOL_TEST_WAIT while slept <= walltime_exceeded: result = func() @@ -306,7 +316,7 @@ class GalaxyInteractorApi(object): output_id = output_data return output_id - def stage_data_async(self, test_data, history_id, tool_id, force_path_paste=False): + def stage_data_async(self, test_data, history_id, tool_id, force_path_paste=False, maxseconds=DEFAULT_TOOL_TEST_WAIT): fname = test_data['fname'] tool_input = { "file_type": test_data['ftype'], @@ -364,7 +374,7 @@ class GalaxyInteractorApi(object): assert "jobs" in submit_response, "Invalid response from server [%s], expecting jobs in response." % submit_response jobs = submit_response["jobs"] assert len(jobs) > 0, "Invalid response from server [%s], expecting a job." % submit_response - return lambda: self.wait_for_job(jobs[0]["id"], history_id, DEFAULT_TOOL_TEST_WAIT) + return lambda: self.wait_for_job(jobs[0]["id"], history_id, maxseconds=maxseconds) def run_tool(self, testdef, history_id, resource_parameters={}): # We need to handle the case where we've uploaded a valid compressed file since the upload @@ -739,7 +749,13 @@ def verify_tool(tool_id, if test_history is None: test_history = galaxy_interactor.new_history() - stage_data_in_history(galaxy_interactor, tool_id, testdef.test_data(), history=test_history, force_path_paste=force_path_paste) + stage_data_in_history(galaxy_interactor, + tool_id, + testdef.test_data(), + history=test_history, + force_path_paste=force_path_paste, + maxseconds=maxseconds, + ) # Once data is ready, run the tool and check the outputs - record API # input, job info, tool run exception, as well as exceptions related to From 881fd87151095fd15c2ba8580e641ce1c3f6bf9e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Jun 2019 14:25:49 +0200 Subject: [PATCH 67/88] Drop unnecessary guard against None Thanks Nicola! --- lib/galaxy/tools/verify/interactor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 339ac8cc7cc..f91e099c254 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -247,9 +247,7 @@ class GalaxyInteractorApi(object): def wait_for(self, func, **kwd): sleep_amount = 0.2 slept = 0 - walltime_exceeded = kwd.get("maxseconds") - if walltime_exceeded is None: - walltime_exceeded = DEFAULT_TOOL_TEST_WAIT + walltime_exceeded = int(kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT)) while slept <= walltime_exceeded: result = func() From 29cf41b8204f84371dbf2c8fa19f51da457ec754 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 12 Jun 2019 08:33:25 -0400 Subject: [PATCH 68/88] Update packages/containers/galaxy/project_galaxy_containers.py Co-Authored-By: Marius van den Beek --- packages/containers/galaxy/project_galaxy_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/containers/galaxy/project_galaxy_containers.py b/packages/containers/galaxy/project_galaxy_containers.py index cc927a1cdf7..6442c8261bb 100644 --- a/packages/containers/galaxy/project_galaxy_containers.py +++ b/packages/containers/galaxy/project_galaxy_containers.py @@ -6,7 +6,7 @@ PROJECT_NAME = "galaxy-containers" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" PROJECT_URL = "https://github.com/galaxyproject/galaxy" PROJECT_AUTHOR = 'Galaxy Project and Community' -PROJECT_DESCRIPTION = 'Galaxy Container Modelling and Interaction Abstractions' +PROJECT_DESCRIPTION = 'Galaxy Container Modeling and Interaction Abstractions' PROJECT_EMAIL = 'jmchilton@gmail.com' RAW_CONTENT_URL = "https://raw.github.com/%s/%s/master/" % ( PROJECT_USERAME, PROJECT_NAME From 974096c3f968e702a223cd20140e75392d575c13 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 12 Jun 2019 08:50:26 -0400 Subject: [PATCH 69/88] Fix loading job_conf from YAML files. I think previous testing of this dependency handling stuff was just with embedded job conf. --- lib/galaxy/dependencies/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 274e1c69a80..e3c614ecdb0 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -69,7 +69,8 @@ class ConditionalDependencies(object): pass else: try: - job_conf_dict = yaml.safe_load(job_conf_path) + with open("job_conf_path", "r") as f: + job_conf_dict = yaml.safe_load(f) load_job_config_dict(job_conf_dict) except (OSError, IOError): pass From 4aa0472760f822a06254f850fe4d6dce8486d0a6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 7 Jun 2019 10:01:11 -0400 Subject: [PATCH 70/88] Fix Pulsar job files API to allow transferring directory content. (cherry picked from commit 1ce7b2a1af72d277001f9c3e64dd3f1188ae3b5d) --- lib/galaxy/webapps/galaxy/api/job_files.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/job_files.py b/lib/galaxy/webapps/galaxy/api/job_files.py index 813db8bca9c..5f571b94049 100644 --- a/lib/galaxy/webapps/galaxy/api/job_files.py +++ b/lib/galaxy/webapps/galaxy/api/job_files.py @@ -99,6 +99,8 @@ class JobFilesAPIController(BaseAPIController): else: input_file = payload.get("file", payload.get("__file", None)).file + target_dir = os.path.dirname(path) + util.safe_makedirs(target_dir) try: shutil.move(input_file.name, path) finally: From fd60eccef85ba7c11d3a2ba2e15aabb3dea512bf Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 12 Jun 2019 14:26:40 +0100 Subject: [PATCH 71/88] Set default `maxseconds` to `DEFAULT_TOOL_TEST_WAIT` for `verify_tool()` --- lib/galaxy/tools/verify/interactor.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index f91e099c254..757fe2047b6 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -733,15 +733,14 @@ def verify_tool(tool_id, quiet=False, test_history=None, force_path_paste=False, - maxseconds=None, + maxseconds=DEFAULT_TOOL_TEST_WAIT, tool_test_dicts=None): if resource_parameters is None: resource_parameters = {} tool_test_dicts = tool_test_dicts or galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) tool_test_dict = tool_test_dicts[test_index] + tool_test_dict.setdefault('maxseconds', maxseconds) testdef = ToolTestDescription(tool_test_dict) - if maxseconds is not None: - testdef.maxseconds = int(maxseconds) _handle_def_errors(testdef) if test_history is None: @@ -752,8 +751,7 @@ def verify_tool(tool_id, testdef.test_data(), history=test_history, force_path_paste=force_path_paste, - maxseconds=maxseconds, - ) + maxseconds=maxseconds) # Once data is ready, run the tool and check the outputs - record API # input, job info, tool run exception, as well as exceptions related to From 63a7134cbb46cf9d5debf99db4d7280cff2ed8e2 Mon Sep 17 00:00:00 2001 From: janani Date: Wed, 12 Jun 2019 17:25:59 +0200 Subject: [PATCH 72/88] round cpu exec time --- lib/galaxy/job_metrics/formatting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/job_metrics/formatting.py b/lib/galaxy/job_metrics/formatting.py index 6bda18418c2..0e0ce11fbc3 100644 --- a/lib/galaxy/job_metrics/formatting.py +++ b/lib/galaxy/job_metrics/formatting.py @@ -11,8 +11,8 @@ class JobMetricFormatter(object): def seconds_to_str(value): """Convert seconds to a simple simple string describing the amount of time.""" if value < 60: - return "%s seconds" % value + return "%s seconds" % round(value, 2) elif value < 3600: - return "%s minutes" % (value / 60) + return "%s minutes" % round(value / 60, 2) else: - return "%s hours and %s minutes" % (value / 3600, (value % 3600) / 60) + return "%s hours and %s minutes" % (round(value / 3600, 2), round((value % 3600) / 60, 2)) From f3c310aa85f145876a9c2f5a69ae8cac786c3b45 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 12 Jun 2019 08:57:41 -0400 Subject: [PATCH 73/88] Use Python 3 for package/ unit testing. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a3cee44d75..7cac35497e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -156,7 +156,7 @@ jobs: - run: tox -e validate_test_tools test_galaxy_packages: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:3.5 <<: *set_workdir steps: - *restore_repo_cache From 811ab0b54797d80a92bf219d4bf9a8ae0f53f54a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 12 Jun 2019 10:22:26 -0400 Subject: [PATCH 74/88] Fix jstree unit tests for Python 3. --- lib/galaxy/util/jstree.py | 9 +++------ run_tests.sh | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/util/jstree.py b/lib/galaxy/util/jstree.py index 49906c52db9..d1682306a57 100644 --- a/lib/galaxy/util/jstree.py +++ b/lib/galaxy/util/jstree.py @@ -47,22 +47,19 @@ class Node(dictobj.DictionaryObject): For example, users may want to pass "attr" or some other valid jsTree options. Example: - >>> import jstree - >>> node = jstree.Node('a', None) + >>> node = Node('a', None) >>> print(node) Node({'text': 'a', 'children': MutableDictionaryObject({})}) >>> print(node.jsonData()) {'text': 'a'} - >>> import jstree - >>> node = jstree.Node('a', 1) + >>> node = Node('a', 1) >>> print(node) Node({'text': 'a', 'children': MutableDictionaryObject({}), 'li_attr': DictionaryObject({'id': 1}), 'id': 1}) >>> print(node.jsonData()) {'text': 'a', 'id': 1, 'li_attr': {'id': 1}} - >>> import jstree - >>> node = jstree.Node('a', 5, icon="folder", state = {'opened': True}) + >>> node = Node('a', 5, icon="folder", state = {'opened': True}) >>> print(node) Node({'text': 'a', 'id': 5, 'state': DictionaryObject({'opened': True}), 'children': MutableDictionaryObject({}), 'li_attr': DictionaryObject({'id': 5}), 'icon': 'folder'}) >>> print(node.jsonData()) diff --git a/run_tests.sh b/run_tests.sh index 0efac9e7111..6f94c4b273a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -515,7 +515,7 @@ do -u|-unit|--unit) report_file="run_unit_tests.html" test_script="pytest" - unit_extra='--doctest-modules --ignore lib/galaxy/web/proxy/js/node_modules/ --ignore lib/galaxy/webapps/tool_shed/controllers --ignore lib/galaxy/jobs/runners/chronos.py --ignore lib/galaxy/webapps/tool_shed/model/migrate --ignore lib/galaxy/util/jstree.py' + unit_extra='--doctest-modules --ignore lib/galaxy/web/proxy/js/node_modules/ --ignore lib/galaxy/webapps/tool_shed/controllers --ignore lib/galaxy/jobs/runners/chronos.py --ignore lib/galaxy/webapps/tool_shed/model/migrate' if [ $# -gt 1 ]; then unit_extra="$unit_extra $2" shift 2 From 223b4cb5a6bd09f959aff999f4277cd4638f0dc2 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 12 Jun 2019 16:22:33 +0100 Subject: [PATCH 75/88] Use assert in doctests --- lib/galaxy/util/jstree.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/util/jstree.py b/lib/galaxy/util/jstree.py index d1682306a57..f5b3001c939 100644 --- a/lib/galaxy/util/jstree.py +++ b/lib/galaxy/util/jstree.py @@ -48,22 +48,16 @@ class Node(dictobj.DictionaryObject): Example: >>> node = Node('a', None) - >>> print(node) - Node({'text': 'a', 'children': MutableDictionaryObject({})}) - >>> print(node.jsonData()) - {'text': 'a'} + >>> assert node._items == {'text': 'a', 'children': dictobj.MutableDictionaryObject({})} + >>> assert node.jsonData() == {'text': 'a'} >>> node = Node('a', 1) - >>> print(node) - Node({'text': 'a', 'children': MutableDictionaryObject({}), 'li_attr': DictionaryObject({'id': 1}), 'id': 1}) - >>> print(node.jsonData()) - {'text': 'a', 'id': 1, 'li_attr': {'id': 1}} + >>> assert node._items == {'text': 'a', 'children': dictobj.MutableDictionaryObject({}), 'li_attr': dictobj.DictionaryObject({'id': 1}), 'id': 1} + >>> assert node.jsonData() == {'text': 'a', 'id': 1, 'li_attr': {'id': 1}} >>> node = Node('a', 5, icon="folder", state = {'opened': True}) - >>> print(node) - Node({'text': 'a', 'id': 5, 'state': DictionaryObject({'opened': True}), 'children': MutableDictionaryObject({}), 'li_attr': DictionaryObject({'id': 5}), 'icon': 'folder'}) - >>> print(node.jsonData()) - {'text': 'a', 'state': {'opened': True}, 'id': 5, 'li_attr': {'id': 5}, 'icon': 'folder'} + >>> assert node._items == {'text': 'a', 'id': 5, 'state': dictobj.DictionaryObject({'opened': True}), 'children': dictobj.MutableDictionaryObject({}), 'li_attr': dictobj.DictionaryObject({'id': 5}), 'icon': 'folder'} + >>> assert node.jsonData() == {'text': 'a', 'state': {'opened': True}, 'id': 5, 'li_attr': {'id': 5}, 'icon': 'folder'} """ super(Node, self).__init__() From 5b51806ca865ad65e8155553e86aa4603fece953 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 12 Jun 2019 15:24:41 -0400 Subject: [PATCH 76/88] Py3 type fixes in the Slurm runner --- lib/galaxy/jobs/runners/slurm.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index ec2ce131d45..9b5b3b7600b 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -10,6 +10,7 @@ import time from galaxy import model from galaxy.jobs.runners.drmaa import DRMAAJobRunner +from galaxy.util import unicodify from galaxy.util.logging import get_logger log = get_logger(__name__) @@ -51,7 +52,7 @@ class SlurmJobRunner(DRMAAJobRunner): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() if p.returncode != 0: - stderr = stderr.strip() + stderr = unicodify(stderr).strip() if stderr == 'SLURM accounting storage is disabled': log.warning('SLURM accounting storage is not properly configured, unable to run sacct') return @@ -59,7 +60,7 @@ class SlurmJobRunner(DRMAAJobRunner): # First line is for 'job_id' # Second line is for 'job_id.batch' (only available after the batch job is complete) # Following lines are for the steps 'job_id.0', 'job_id.1', ... (but Galaxy does not use steps) - first_line = stdout.splitlines()[0] + first_line = unicodify(stdout).splitlines()[0] # Strip whitespaces and the final '+' (if present), only return the first word return first_line.strip().rstrip('+').split()[0] @@ -77,6 +78,7 @@ class SlurmJobRunner(DRMAAJobRunner): stdout, stderr = p.communicate() if p.returncode != 0: # Will need to be more clever here if this message is not consistent + stderr = unicodify(stderr) if stderr == 'slurm_load_jobs error: Invalid job id specified\n': # The job may be old, try to get its state with sacct job_state = _get_slurm_state_with_sacct(job_id, cluster) @@ -84,7 +86,7 @@ class SlurmJobRunner(DRMAAJobRunner): return job_state return 'NOT_FOUND' raise Exception('`%s` returned %s, stderr: %s' % (' '.join(cmd), p.returncode, stderr)) - stdout = stdout.strip() + stdout = unicodify(stdout).strip() # stdout is a single line in format "key1=value1 key2=value2 ..." job_info_keys = [] job_info_values = [] From 050f591dbf802065699d594c226dddc1bb931b53 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 13 Jun 2019 11:14:35 +0100 Subject: [PATCH 77/88] Update ``parse_tests()`` docstring `ToolTestBuilder` has been replaced by `ToolTestDescription` in commit c3af6c8feae90bc2d4b6328559fb79e3d7d35a5c . --- lib/galaxy/tools/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index a4acd0f5ee2..e8c74b6e725 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -21,7 +21,7 @@ log = logging.getLogger(__name__) @nottest def parse_tests(tool, tests_source): """ - Build ToolTestBuilder objects for each "" elements and + Build ToolTestDescription objects for each "" elements and return default interactor (if any). """ raw_tests_dict = tests_source.parse_tests_to_dict() From 2fdc1c53ca90fbed35dcb9444c3c4d458f5e3abc Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 13 Jun 2019 12:24:17 +0100 Subject: [PATCH 78/88] Run more CircleCI jobs using the ``python:3.5`` image Also run `test_galaxy_packages` job through tox. --- .ci/validate_test_tools.sh | 4 ++-- .circleci/config.yml | 19 ++++++++++--------- packages/test.sh | 9 ++++----- tox.ini | 5 ++++- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.ci/validate_test_tools.sh b/.ci/validate_test_tools.sh index 9b43e7a6c46..258799dc95d 100755 --- a/.ci/validate_test_tools.sh +++ b/.ci/validate_test_tools.sh @@ -1,11 +1,11 @@ #!/bin/sh -cd `dirname $0`/.. +cd "$(dirname "$0")"/.. xsd_path="lib/galaxy/tools/xsd/galaxy.xsd" # Lint the XSD xmllint --noout "$xsd_path" test_tools_path='test/functional/tools' -tool_files_list=$(ls "$test_tools_path"/*xml | grep -v '_conf.xml$') +tool_files_list=$(ls "$test_tools_path"/*.xml | grep -v '_conf.xml$') sh scripts/validate_tools.sh $tool_files_list diff --git a/.circleci/config.yml b/.circleci/config.yml index 7cac35497e7..32dfbce7036 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ variables: jobs: get_code: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:3.5 <<: *set_workdir steps: # Replace standard code checkout with shallow clone to speed things up. @@ -83,7 +83,7 @@ jobs: - ~/repo py27_lint: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:2.7 <<: *set_workdir steps: - *restore_repo_cache @@ -91,7 +91,7 @@ jobs: - run: tox -e py27-lint py27_unit: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:2.7 <<: *set_workdir steps: - *restore_repo_cache @@ -99,7 +99,7 @@ jobs: - run: tox -e py27-unit py27_docstring: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:2.7 <<: *set_workdir steps: - *restore_repo_cache @@ -107,7 +107,7 @@ jobs: - run: tox -e py27-lint_docstring_include_list py27_first_startup: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:2.7 <<: *set_workdir steps: - *restore_repo_cache @@ -139,14 +139,14 @@ jobs: <<: *set_workdir steps: - *restore_repo_cache - - *install_tox - run: sudo apt-get update # For uwsgi - run: sudo apt-get install -y libpython3.5-dev + - *install_tox - run: tox -e py35-first_startup validate_test_tools: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:3.5 <<: *set_workdir steps: - *restore_repo_cache @@ -160,10 +160,11 @@ jobs: <<: *set_workdir steps: - *restore_repo_cache - - run: bash packages/test.sh + - *install_tox + - run: tox -e test_galaxy_packages check_py3_compatibility: docker: - - image: circleci/python:2.7.15 + - image: circleci/python:3.5 <<: *set_workdir steps: - *restore_repo_cache diff --git a/packages/test.sh b/packages/test.sh index 5deb29bf919..d311a4d5ed1 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -5,8 +5,9 @@ set -e # Change to packages directory. cd "$(dirname "$0")" +# Use a throw-away virtualenv TEST_PYTHON=${TEST_PYTHON:-"python"} -TEST_ENV_DIR=${TEST_ENV_DIR:-`mktemp -d -t gxpkgtestenvXXXXXX`} +TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" @@ -31,13 +32,11 @@ for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do cd "$package_dir" pip install -e . - if [ "$package_dir" = "util" ]; - then + if [ "$package_dir" = "util" ]; then pip install -e '.[template,jstree]' fi - if [[ "$run_tests" == "1" ]]; - then + if [[ "$run_tests" == "1" ]]; then pytest --doctest-modules galaxy tests fi cd .. diff --git a/tox.ini b/tox.ini index 9904d2a82f1..0aec466cf97 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] # envlist is the list of environments that are tested when `tox` is run without any option # hyphens in an environment name are used to delimit factors -envlist = check_py3_compatibility, py{27,35}-first_startup, py{27,35}-lint, py27-lint_docstring_include_list, py{27,35}-unit, validate_test_tools +envlist = check_py3_compatibility, py{27,35}-first_startup, py{27,35}-lint, py27-lint_docstring_include_list, py{27,35}-unit, test_galaxy_packages, validate_test_tools skipsdist = True [testenv] @@ -35,6 +35,9 @@ commands = bash .ci/flake8_wrapper_docstrings.sh --exclude [testenv:py27-lint_docstring_include_list] commands = bash .ci/flake8_wrapper_docstrings.sh --include +[testenv:test_galaxy_packages] +commands = bash packages/test.sh + [testenv:validate_test_tools] commands = bash .ci/validate_test_tools.sh From 6013419adce2583c00ebffa292b9d87263b8737e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 13 Jun 2019 12:39:21 +0100 Subject: [PATCH 79/88] Fix ``--coverage`` option of ``run_tests.sh`` for pytest-based tests. Also: - do not create a giant XML coverage report file - update other dependencies using `make update-dependencies`. Introduced in commit 1152a19fd2cd5d6589becde3c7ebdd6435a33cff . --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 1 + .../pipfiles/default/pinned-dev-requirements.txt | 8 +++++--- .../pipfiles/default/pinned-requirements.txt | 4 ++-- run_tests.sh | 13 ++----------- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 02777c6b6b6..0a0bb133704 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -15,6 +15,7 @@ mock = "*" NoseHTML = "*" pygithub3 = {version = "*", markers = "python_version < '3'"} pytest = "*" +pytest-cov = "*" pytest-html = "*" pytest-pythonpath = "*" pytest-postgresql = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index c95d3ff3274..1863f4aad97 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -10,13 +10,14 @@ chardet==3.0.4 commonmark==0.9.0 configparser==3.7.4 ; python_version < '3.2' contextlib2==0.5.5 ; python_version < '3.5' +coverage==4.5.3 docutils==0.14 funcsigs==1.0.2 ; python_version < '3.3' future==0.17.1 gunicorn==19.9.0 idna==2.8 imagesize==1.1.0 -importlib-metadata==0.17 +importlib-metadata==0.18 jinja2==2.10.1 lxml==4.3.4 markdown==2.6.11 @@ -31,16 +32,17 @@ pathlib2==2.3.2 ; python_version < '3' pathtools==0.1.2 pluggy==0.12.0 port-for==0.4 -psutil==5.6.2 +psutil==5.6.3 py==1.8.0 pygithub3==0.5.1 ; python_version < '3' pygments==2.4.2 pyparsing==2.4.0 +pytest-cov==2.7.1 pytest-html==1.20.0 pytest-metadata==1.8.0 pytest-postgresql==1.4.1 pytest-pythonpath==0.7.3 -pytest==4.6.2 +pytest==4.6.3 pytz==2019.1 pyyaml==5.1.1 recommonmark==0.5.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index a8afc0745a0..4bc74ec1fe7 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -29,7 +29,7 @@ bleach==3.1.0 boltons==19.1.0 boto3==1.9.114 boto==2.49.0 -botocore==1.12.165 +botocore==1.12.167 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 @@ -118,7 +118,7 @@ pathlib2==2.3.2 ; python_version < '3' pbr==5.2.1 prettytable==0.7.2 prov==1.5.1 -psutil==5.6.2 +psutil==5.6.3 pulsar-galaxy-lib==0.12.1 pyasn1-modules==0.2.5 pyasn1==0.4.5 diff --git a/run_tests.sh b/run_tests.sh index 6f94c4b273a..61a5d8bcd62 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -374,7 +374,6 @@ do api_script="./test/api" shift 1 fi - coverage_file="api_coverage.xml" ;; -selenium|--selenium) GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" @@ -428,7 +427,6 @@ do marker="-m tool" test_script="pytest" report_file="run_framework_tests.html" - coverage_file="framework_coverage.xml" framework_test=1; shift 1 ;; @@ -437,7 +435,6 @@ do marker="-m tool" test_script="pytest" report_file="run_framework_tests.html" - coverage_file="main_tools_coverage.xml" framework_test=1; shift 1 ;; @@ -445,7 +442,6 @@ do marker="-m data_manager" test_script="pytest" report_file="run_data_managers_tests.html" - coverage_file="data_managers_coverage.xml" data_managers_test=1; shift 1 ;; @@ -454,7 +450,6 @@ do marker="-m tool" test_script="pytest" report_file="run_migrated_tests.html" - coverage_file="migrated_coverage.xml" migrated_test=1; shift ;; @@ -463,7 +458,6 @@ do marker="-m tool" test_script="pytest" report_file="run_installed_tests.html" - coverage_file="installed_coverage.xml" installed_test=1; shift ;; @@ -503,7 +497,6 @@ do # Must have coverage installed (try `which coverage`) - only valid with --unit # for now. Would be great to get this to work with functional tests though. coverage_arg="--with-coverage" - NOSE_WITH_COVERAGE=true shift ;; --debug) @@ -523,7 +516,6 @@ do unit_extra="$unit_extra lib test/unit" shift 1 fi - coverage_file="unit_coverage.xml" ;; -i|-integration|--integration) GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" @@ -535,7 +527,6 @@ do else integration_extra="./test/integration" shift 1 - coverage_file="integration_coverage.xml" fi ;; --no_cleanup) @@ -645,8 +636,8 @@ else fi export GALAXY_TEST_TOOL_CONF if [ "$test_script" = 'pytest' ]; then - if [ "$coverage_arg" = "--with_coverage" ]; then - coverage_arg="--cov-report term --cov-report xml:cov-unit.xml --cov=lib" + if [ "$coverage_arg" = '--with-coverage' ]; then + coverage_arg="--cov-report term --cov=lib" fi "$test_script" -v --html "$report_file" $coverage_arg $xunit_args $extra_args "$@" else From 421eca8eb76deabbd3457f6dc7199180de5049ae Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 13 Jun 2019 12:49:41 +0100 Subject: [PATCH 80/88] Fix ``--external_url`` and related options Introduced in commit eac6c10ec93a6f5a610a9d07d1e7601049f4f7cf . --- run_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 61a5d8bcd62..a72f96257bb 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -411,15 +411,15 @@ do shift ;; --external_url) - GALAXY_TEST_EXTERNAL=$2 + export GALAXY_TEST_EXTERNAL=$2 shift 2 ;; --external_master_key) - GALAXY_CONFIG_MASTER_KEY=$2 + export GALAXY_CONFIG_MASTER_API_KEY=$2 shift 2 ;; --external_user_key) - GALAXY_TEST_USER_API_KEY=$2 + export GALAXY_TEST_USER_API_KEY=$2 shift 2 ;; -f|-framework|--framework) From 90a6c6d46bbc630b36c414e3e598455dea94459e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 13 Jun 2019 12:53:26 +0100 Subject: [PATCH 81/88] Fix issues reported by ShellCheck --- create_db.sh | 4 ++-- extract_dataset_parts.sh | 4 ++-- run_tests.sh | 9 ++++----- scripts/cleanup_datasets/delete_datasets.sh | 4 ++-- .../delete_userless_histories.sh | 4 ++-- scripts/cleanup_datasets/populate_uuid.sh | 4 ++-- scripts/cleanup_datasets/purge_datasets.sh | 4 ++-- scripts/cleanup_datasets/purge_folders.sh | 4 ++-- scripts/cleanup_datasets/purge_histories.sh | 4 ++-- scripts/cleanup_datasets/purge_libraries.sh | 4 ++-- scripts/cleanup_datasets/update_metadata.sh | 2 +- scripts/maintenance.sh | 4 ++-- scripts/migrate_tools/0002_tools.sh | 4 ++-- scripts/migrate_tools/0003_tools.sh | 4 ++-- scripts/migrate_tools/0004_tools.sh | 4 ++-- scripts/migrate_tools/0005_tools.sh | 4 ++-- scripts/migrate_tools/0006_tools.sh | 4 ++-- scripts/migrate_tools/0007_tools.sh | 4 ++-- scripts/migrate_tools/0008_tools.sh | 4 ++-- scripts/migrate_tools/0009_tools.sh | 4 ++-- scripts/migrate_tools/0010_tools.sh | 4 ++-- scripts/migrate_tools/0011_tools.sh | 4 ++-- scripts/migrate_tools/0012_tools.sh | 4 ++-- .../tool_shed/migrate_tools_to_repositories.sh | 2 +- scripts/validate_tools.sh | 16 +++++++--------- 25 files changed, 55 insertions(+), 58 deletions(-) diff --git a/create_db.sh b/create_db.sh index 8c77eff858d..3c34dd6c3e5 100755 --- a/create_db.sh +++ b/create_db.sh @@ -1,9 +1,9 @@ #!/bin/sh -cd `dirname $0` +cd "$(dirname "$0")" . ./scripts/common_startup_functions.sh setup_python -python ./scripts/create_db.py $@ +python ./scripts/create_db.py "$@" diff --git a/extract_dataset_parts.sh b/extract_dataset_parts.sh index f17ced8bcd5..7591b4a5ae3 100755 --- a/extract_dataset_parts.sh +++ b/extract_dataset_parts.sh @@ -1,6 +1,6 @@ #!/bin/sh -cd `dirname $0` +cd "$(dirname "$0")" . ./scripts/common_startup_functions.sh @@ -9,5 +9,5 @@ setup_python for file in $1/split_info*.json do # echo processing $file - python ./scripts/extract_dataset_part.py $file + python ./scripts/extract_dataset_part.py "$file" done diff --git a/run_tests.sh b/run_tests.sh index a72f96257bb..1236e8742cc 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,6 @@ #!/bin/sh -pwd_dir=$(pwd) -cd `dirname $0` +cd "$(dirname "$0")" rm -f run_functional_tests.log @@ -318,8 +317,8 @@ then -e "GALAXY_TEST_DATABASE_TYPE=$db_type" \ -e "LC_ALL=C" \ --rm \ - -v `pwd`:/galaxy \ - -v `pwd`/test/docker/base/run_test_wrapper.sh:/usr/local/bin/run_test_wrapper.sh $DOCKER_IMAGE "$@" + -v "$(pwd)":/galaxy \ + -v "$(pwd)"/test/docker/base/run_test_wrapper.sh:/usr/local/bin/run_test_wrapper.sh "$DOCKER_IMAGE" "$@" exit $? fi @@ -607,7 +606,7 @@ elif [ -n "$toolshed_script" ]; then elif [ -n "$api_script" ]; then extra_args="$api_script" elif [ -n "$section_id" ]; then - extra_args=`python tool_list.py $section_id` + extra_args=$(python tool_list.py "$section_id") elif [ -n "$unit_extra" ]; then extra_args="$unit_extra" elif [ -n "$integration_extra" ]; then diff --git a/scripts/cleanup_datasets/delete_datasets.sh b/scripts/cleanup_datasets/delete_datasets.sh index 501c234a8d5..e7dfb3f6874 100644 --- a/scripts/cleanup_datasets/delete_datasets.sh +++ b/scripts/cleanup_datasets/delete_datasets.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -6 -r $@ >> ./scripts/cleanup_datasets/delete_datasets.log +cd "$(dirname "$0")"/../.. +python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -6 -r "$@" >> ./scripts/cleanup_datasets/delete_datasets.log diff --git a/scripts/cleanup_datasets/delete_userless_histories.sh b/scripts/cleanup_datasets/delete_userless_histories.sh index 72d68db72ec..9bb8c70882f 100644 --- a/scripts/cleanup_datasets/delete_userless_histories.sh +++ b/scripts/cleanup_datasets/delete_userless_histories.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -1 $@ >> ./scripts/cleanup_datasets/delete_userless_histories.log +cd "$(dirname "$0")"/../.. +python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -1 "$@" >> ./scripts/cleanup_datasets/delete_userless_histories.log diff --git a/scripts/cleanup_datasets/populate_uuid.sh b/scripts/cleanup_datasets/populate_uuid.sh index ac2028462cf..c12c5240c19 100644 --- a/scripts/cleanup_datasets/populate_uuid.sh +++ b/scripts/cleanup_datasets/populate_uuid.sh @@ -1,5 +1,5 @@ #!/bin/sh -cd `dirname $0`/../.. +cd "$(dirname "$0")"/../.. export PYTHONPATH=./lib/ -python ./scripts/cleanup_datasets/populate_uuid.py ./config/galaxy.ini $@ +python ./scripts/cleanup_datasets/populate_uuid.py ./config/galaxy.ini "$@" diff --git a/scripts/cleanup_datasets/purge_datasets.sh b/scripts/cleanup_datasets/purge_datasets.sh index e4405015c50..6d5c12bb880 100644 --- a/scripts/cleanup_datasets/purge_datasets.sh +++ b/scripts/cleanup_datasets/purge_datasets.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -3 -r $@ >> ./scripts/cleanup_datasets/purge_datasets.log +cd "$(dirname "$0")"/../.. +python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -3 -r "$@" >> ./scripts/cleanup_datasets/purge_datasets.log diff --git a/scripts/cleanup_datasets/purge_folders.sh b/scripts/cleanup_datasets/purge_folders.sh index ff970d0ac8e..d02b7671e6f 100644 --- a/scripts/cleanup_datasets/purge_folders.sh +++ b/scripts/cleanup_datasets/purge_folders.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -5 -r $@ >> ./scripts/cleanup_datasets/purge_folders.log +cd "$(dirname "$0")"/../.. +python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -5 -r "$@" >> ./scripts/cleanup_datasets/purge_folders.log diff --git a/scripts/cleanup_datasets/purge_histories.sh b/scripts/cleanup_datasets/purge_histories.sh index 66e9f3d285f..8c7ff274d4a 100644 --- a/scripts/cleanup_datasets/purge_histories.sh +++ b/scripts/cleanup_datasets/purge_histories.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -2 -r $@ >> ./scripts/cleanup_datasets/purge_histories.log +cd "$(dirname "$0")"/../.. +python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -2 -r "$@" >> ./scripts/cleanup_datasets/purge_histories.log diff --git a/scripts/cleanup_datasets/purge_libraries.sh b/scripts/cleanup_datasets/purge_libraries.sh index 6a96b30268d..9e66e05cb5c 100644 --- a/scripts/cleanup_datasets/purge_libraries.sh +++ b/scripts/cleanup_datasets/purge_libraries.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -4 -r $@ >> ./scripts/cleanup_datasets/purge_libraries.log +cd "$(dirname "$0")"/../.. +python ./scripts/cleanup_datasets/cleanup_datasets.py -d 10 -4 -r "$@" >> ./scripts/cleanup_datasets/purge_libraries.log diff --git a/scripts/cleanup_datasets/update_metadata.sh b/scripts/cleanup_datasets/update_metadata.sh index 7933a0d9ebb..a78325d8ee7 100644 --- a/scripts/cleanup_datasets/update_metadata.sh +++ b/scripts/cleanup_datasets/update_metadata.sh @@ -6,4 +6,4 @@ . ./scripts/get_python.sh . ./setup_paths.sh -$GALAXY_PYTHON ./scripts/cleanup_datasets/update_metadata.py ./config/galaxy.ini $@ +"$GALAXY_PYTHON" ./scripts/cleanup_datasets/update_metadata.py ./config/galaxy.ini "$@" diff --git a/scripts/maintenance.sh b/scripts/maintenance.sh index 8ca4771ed54..e45d11bb603 100755 --- a/scripts/maintenance.sh +++ b/scripts/maintenance.sh @@ -6,7 +6,7 @@ set -e display_help(){ - scriptname=`basename $0` + scriptname=$(basename "$0") printf "./$scriptname [--help] [--no-dry-run] [--days 10] Will run the galaxy cleanup scripts in the recommend order. By default a 'dry-run' is started. Specify --no-dry-run to do the actual cleanup. @@ -52,7 +52,7 @@ do esac done -cd "$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"/../ +cd "$(dirname "$0")"/.. . scripts/common_startup_functions.sh diff --git a/scripts/migrate_tools/0002_tools.sh b/scripts/migrate_tools/0002_tools.sh index 926aba9529d..7262196b5a9 100644 --- a/scripts/migrate_tools/0002_tools.sh +++ b/scripts/migrate_tools/0002_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0002_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0002_tools.xml "$@" diff --git a/scripts/migrate_tools/0003_tools.sh b/scripts/migrate_tools/0003_tools.sh index dfc3bde56b3..f8a1ba8bb16 100644 --- a/scripts/migrate_tools/0003_tools.sh +++ b/scripts/migrate_tools/0003_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0003_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0003_tools.xml "$@" diff --git a/scripts/migrate_tools/0004_tools.sh b/scripts/migrate_tools/0004_tools.sh index 40b76956fa2..7ae608f3683 100644 --- a/scripts/migrate_tools/0004_tools.sh +++ b/scripts/migrate_tools/0004_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0004_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0004_tools.xml "$@" diff --git a/scripts/migrate_tools/0005_tools.sh b/scripts/migrate_tools/0005_tools.sh index c9f1f3c61d6..da7f9c09098 100644 --- a/scripts/migrate_tools/0005_tools.sh +++ b/scripts/migrate_tools/0005_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0005_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0005_tools.xml "$@" diff --git a/scripts/migrate_tools/0006_tools.sh b/scripts/migrate_tools/0006_tools.sh index 1f000ebeca6..ff85522a5f9 100644 --- a/scripts/migrate_tools/0006_tools.sh +++ b/scripts/migrate_tools/0006_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0006_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0006_tools.xml "$@" diff --git a/scripts/migrate_tools/0007_tools.sh b/scripts/migrate_tools/0007_tools.sh index a6cf948593d..8a88f44420c 100644 --- a/scripts/migrate_tools/0007_tools.sh +++ b/scripts/migrate_tools/0007_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0007_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0007_tools.xml "$@" diff --git a/scripts/migrate_tools/0008_tools.sh b/scripts/migrate_tools/0008_tools.sh index 50cafd19936..d312237023b 100644 --- a/scripts/migrate_tools/0008_tools.sh +++ b/scripts/migrate_tools/0008_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0008_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0008_tools.xml "$@" diff --git a/scripts/migrate_tools/0009_tools.sh b/scripts/migrate_tools/0009_tools.sh index 36f162b141c..78242fe2422 100644 --- a/scripts/migrate_tools/0009_tools.sh +++ b/scripts/migrate_tools/0009_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0009_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0009_tools.xml "$@" diff --git a/scripts/migrate_tools/0010_tools.sh b/scripts/migrate_tools/0010_tools.sh index fde17704dd3..03c65dd9973 100644 --- a/scripts/migrate_tools/0010_tools.sh +++ b/scripts/migrate_tools/0010_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0010_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0010_tools.xml "$@" diff --git a/scripts/migrate_tools/0011_tools.sh b/scripts/migrate_tools/0011_tools.sh index a31365804fd..e06a6be40b3 100644 --- a/scripts/migrate_tools/0011_tools.sh +++ b/scripts/migrate_tools/0011_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0011_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0011_tools.xml "$@" diff --git a/scripts/migrate_tools/0012_tools.sh b/scripts/migrate_tools/0012_tools.sh index 97fbe354d04..2df312f980a 100644 --- a/scripts/migrate_tools/0012_tools.sh +++ b/scripts/migrate_tools/0012_tools.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. -python ./scripts/migrate_tools/migrate_tools.py 0012_tools.xml $@ +cd "$(dirname "$0")"/../.. +python ./scripts/migrate_tools/migrate_tools.py 0012_tools.xml "$@" diff --git a/scripts/tool_shed/migrate_tools_to_repositories.sh b/scripts/tool_shed/migrate_tools_to_repositories.sh index e1108e20307..bbab9b1b333 100644 --- a/scripts/tool_shed/migrate_tools_to_repositories.sh +++ b/scripts/tool_shed/migrate_tools_to_repositories.sh @@ -1,4 +1,4 @@ #!/bin/sh -cd `dirname $0`/../.. +cd "$(dirname "$0")"/../.. python ./scripts/tool_shed/migrate_tools_to_repositories.py ./community_wsgi.ini >> ./scripts/tool_shed/migrate_tools_to_repositories.log diff --git a/scripts/validate_tools.sh b/scripts/validate_tools.sh index 11a52c2f0aa..f57c1cb814a 100755 --- a/scripts/validate_tools.sh +++ b/scripts/validate_tools.sh @@ -1,33 +1,31 @@ #!/bin/sh -cd `dirname $0`/.. +cd "$(dirname "$0")"/.. GALAXY_SKIP_CLIENT_BUILD=1 ./scripts/common_startup.sh GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-.venv}" -if [ -d "$GALAXY_VIRTUAL_ENV" ]; -then +if [ -d "$GALAXY_VIRTUAL_ENV" ]; then printf "Activating virtualenv at $GALAXY_VIRTUAL_ENV\n" . "$GALAXY_VIRTUAL_ENV/bin/activate" fi xsd_path="lib/galaxy/tool_util/xsd/galaxy.xsd" -err_tmp=`mktemp` +err_tmp=$(mktemp) count=0 exit=0 for p in "$@"; do count=$((count+1)) path="$p" - echo $path + echo "$path" PYTHONPATH=lib:$PYTHONPATH export PYTHONPATH - result=`python -c "import galaxy.tool_util.loader; import xml.etree; xml.etree.ElementTree.dump(galaxy.tool_util.loader.load_tool('$path').getroot())" | xmllint --nowarning --noout --schema "$xsd_path" - 2> "$err_tmp"` - if [ $? -eq 0 ] - then + python -c "import galaxy.tool_util.loader; import xml.etree; xml.etree.ElementTree.dump(galaxy.tool_util.loader.load_tool('$path').getroot())" | xmllint --nowarning --noout --schema "$xsd_path" - 2> "$err_tmp" + if [ $? -eq 0 ]; then echo "ok $count"; else echo "not ok $count $path"; - cat "$err_tmp" | sed 's/^/ /' + sed 's/^/ /' "$err_tmp" exit=1 fi done From f38ff2e0b1067ca1902d7a505af5aa4700996c03 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 3 May 2019 16:18:21 +0200 Subject: [PATCH 82/88] Change unicode test tool To produce NULL character. This tool fails locally but not if testing against the jenkins framework tests or using external docker-galaxy-stable. I don't understand why. --- test/functional/tools/unicode_stream.xml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/unicode_stream.xml b/test/functional/tools/unicode_stream.xml index 7fb2b00a30d..623b83a61be 100644 --- a/test/functional/tools/unicode_stream.xml +++ b/test/functional/tools/unicode_stream.xml @@ -1,21 +1,26 @@ + echo "\x00" '$out_file1'; +#if $include_null: + echo "\x00" > $out_file1; +#end if +echo '$input1' >> '$out_file1'; cat '$cf'; +echo "\x00"; >&2 cat '$cf'; sh -c "exit $exit" ]]> - ვეპხის ტყაოსანი შოთა რუსთაველი - + ვეპხის ტყაოსანი შოთა რუსთაველი + @@ -34,6 +39,10 @@ sh -c "exit $exit" + + + + From a8f050f65df15cafb99e72375ad111e6baf7df00 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 15 Jun 2019 16:52:24 +0200 Subject: [PATCH 83/88] Strip unicode null from stdout/stderr/tool_version --- lib/galaxy/jobs/__init__.py | 3 ++- lib/galaxy/model/__init__.py | 12 +++++++----- lib/galaxy/util/__init__.py | 21 +++++++++++++++++++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 00f97cf3c39..101c0cda08d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1286,7 +1286,8 @@ class JobWrapper(HasResourceParameters): if not os.path.exists(version_filename): version_filename = self.get_version_string_path_legacy() if os.path.exists(version_filename): - self.version_string = open(version_filename).read() + with open(version_filename, 'rb') as fh: + self.version_string = galaxy.util.shrink_and_unicodify(fh.read()) os.unlink(version_filename) outputs_to_working_directory = util.asbool(self.get_destination_configuration("outputs_to_working_directory", False)) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1448ce6f75e..99bef225138 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -289,11 +289,13 @@ class JobLike(object): def set_streams(self, tool_stdout, tool_stderr, job_stdout=None, job_stderr=None, job_messages=None): def shrink_and_unicodify(what, stream): - stream = galaxy.util.unicodify(stream) or u'' - if (len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE): - stream = galaxy.util.shrink_string_by_size(tool_stdout, galaxy.util.DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) - log.info("%s for %s %d is greater than %s, only a portion will be logged to database", what, type(self), self.id, galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) - return stream + if len(stream) > galaxy.util.DATABASE_MAX_STRING_SIZE: + log.info("%s for %s %d is greater than %s, only a portion will be logged to database", + what, + type(self), + self.id, + galaxy.util.DATABASE_MAX_STRING_SIZE_PRETTY) + return galaxy.util.shrink_and_unicodify(stream) self.tool_stdout = shrink_and_unicodify('tool_stdout', tool_stdout) self.tool_stderr = shrink_and_unicodify('tool_stderr', tool_stderr) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 93c76f36003..ff80f1df795 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -402,6 +402,17 @@ def shrink_stream_by_size(value, size, join_by=b"..", left_larger=True, beginnin return unicodify(rval) +def shrink_and_unicodify(stream): + stream = unicodify(stream, strip_null=True) or u'' + if (len(stream) > DATABASE_MAX_STRING_SIZE): + stream = shrink_string_by_size(stream, + DATABASE_MAX_STRING_SIZE, + join_by="\n..\n", + left_larger=True, + beginning_on_size_error=True) + return stream + + def shrink_string_by_size(value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False): if len(value) > size: len_join_by = len(join_by) @@ -993,7 +1004,7 @@ def roundify(amount, sfs=2): return amount[0:sfs] + '0' * (len(amount) - sfs) -def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): +def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=False): u""" Returns a Unicode string or None. @@ -1008,7 +1019,11 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): >>> s = u'lâtín strìñg'; assert unicodify(s.encode('latin-1')) == u'l\ufffdt\ufffdn str\ufffd\ufffdg' >>> s = u'lâtín strìñg'; assert unicodify(s.encode('latin-1'), error='ignore') == u'ltn strg' """ - if value is None or isinstance(value, text_type): + if value is None: + return value + if isinstance(value, text_type): + if strip_null: + return value.replace('\0', '') return value try: if isinstance(value, bytearray): @@ -1025,6 +1040,8 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace'): msg = "Value '%s' could not be coerced to Unicode" % value log.exception(msg) raise Exception(msg) + if strip_null: + return value.replace('\0', '') return value From af9c924f62a840b206b2e95de28182d6765310ec Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 15 Jun 2019 16:52:41 +0200 Subject: [PATCH 84/88] Strip unicode null from peek --- lib/galaxy/model/__init__.py | 8 ++++++++ lib/galaxy/model/mapping.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 99bef225138..95efaa76ecc 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2402,6 +2402,14 @@ class DatasetInstance(object): self.parent_id = parent_id self.validation_errors = validation_errors + @property + def peek(self): + return self._peek + + @peek.setter + def peek(self, peek): + self._peek = unicodify(peek, strip_null=True) + def update(self): self.update_time = galaxy.model.orm.now.now() diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f7f563c54c8..617e690bd02 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -222,7 +222,7 @@ model.HistoryDatasetAssociation.table = Table( Column("name", TrimmedString(255)), Column("info", TrimmedString(255)), Column("blurb", TrimmedString(255)), - Column("peek", TEXT), + Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), Column("metadata", MetadataType(), key="_metadata"), @@ -505,7 +505,7 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("name", TrimmedString(255), index=True), Column("info", TrimmedString(255)), Column("blurb", TrimmedString(255)), - Column("peek", TEXT), + Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), Column("metadata", MetadataType(), key="_metadata"), From 0fe107ecc694bb40e41da0bc2ce49cc04e3fd54d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 16 Jun 2019 17:25:51 +0200 Subject: [PATCH 85/88] Fix fetch_data on python 3 --- lib/galaxy/datatypes/sniff.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 0e226346304..bf21e846004 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -76,6 +76,8 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source else: # Compressed files must be encoded after they are uncompressed in the upload utility, # while binary files should not be encoded at all. + if isinstance(chunk, text_type): + chunk = chunk.encode(target_encoding, target_error) os.write(fd, chunk) os.close(fd) return filename From ab952886b47ff2ab5bc05c3e73c302d543a0e65b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 16 Jun 2019 17:26:10 +0200 Subject: [PATCH 86/88] Include .xml in job_config_file --- test/integration/test_cli_runners.py | 2 +- test/integration/test_kubernetes_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_cli_runners.py b/test/integration/test_cli_runners.py index 64605642e7a..370c37da79f 100644 --- a/test/integration/test_cli_runners.py +++ b/test/integration/test_cli_runners.py @@ -67,7 +67,7 @@ def cli_job_config(remote_connection, shell_plugin='ParamikoShell', job_plugin=' job_conf_str = job_conf_template.substitute(shell_plugin=shell_plugin, job_plugin=job_plugin, **remote_connection._asdict()) - with tempfile.NamedTemporaryFile(suffix="_slurm_integration_job_conf", mode="w", delete=False) as job_conf: + with tempfile.NamedTemporaryFile(suffix="_slurm_integration_job_conf.xml", mode="w", delete=False) as job_conf: job_conf.write(job_conf_str) return job_conf.name diff --git a/test/integration/test_kubernetes_runner.py b/test/integration/test_kubernetes_runner.py index 766260f599c..34e469ee716 100644 --- a/test/integration/test_kubernetes_runner.py +++ b/test/integration/test_kubernetes_runner.py @@ -111,7 +111,7 @@ def job_config(jobs_directory): tool_directory=TOOL_DIR, k8s_config_path=os.environ.get('GALAXY_TEST_KUBE_CONFIG_PATH', '~/.kube/config'), ) - with tempfile.NamedTemporaryFile(suffix="_kubernetes_integration_job_conf", mode="w", delete=False) as job_conf: + with tempfile.NamedTemporaryFile(suffix="_kubernetes_integration_job_conf.xml", mode="w", delete=False) as job_conf: job_conf.write(job_conf_str) return Config(job_conf.name) From 3c17cc949986a4daf1f37e650b519ec2f8ed1b7a Mon Sep 17 00:00:00 2001 From: vahid Date: Sun, 16 Jun 2019 13:20:19 -0700 Subject: [PATCH 87/88] Update cloudauthz to v0.4.0, and run `make updated-dependencies`. --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../pipfiles/default/pinned-dev-requirements.txt | 2 +- .../pipfiles/default/pinned-requirements.txt | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 0a0bb133704..8980b46000a 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -77,7 +77,7 @@ pyparsing = "*" paramiko = "*" python-genomespaceclient = "<2.0" social_auth_core = {version = "==3.1.0+gx0", extras = ['openidconnect']} -cloudauthz = "<=0.2.0" +cloudauthz = "<=0.4.0" gxformat2 = "*" [requires] diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 1863f4aad97..bb14cf2fbde 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -5,7 +5,7 @@ argh==0.26.2 atomicwrites==1.3.0 attrs==19.1.0 babel==2.7.0 -certifi==2019.3.9 +certifi==2019.6.16 chardet==3.0.4 commonmark==0.9.0 configparser==3.7.4 ; python_version < '3.2' diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 4bc74ec1fe7..8a4e6c136ad 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -29,17 +29,17 @@ bleach==3.1.0 boltons==19.1.0 boto3==1.9.114 boto==2.49.0 -botocore==1.12.167 +botocore==1.12.169 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 cachetools==3.1.1 -certifi==2019.3.9 +certifi==2019.6.16 cffi==1.12.3 chardet==3.0.4 cheetah3==3.2.3 -cliff==2.14.1 -cloudauthz==0.2.0 +cliff==2.15.0 +cloudauthz==0.4.0 cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' @@ -77,7 +77,7 @@ jsonpatch==1.23 jsonpointer==2.0 jsonschema==3.0.1 keystoneauth1==3.14.0 -kombu==4.6.1 +kombu==4.6.3 lockfile==0.12.2 lxml==4.3.4 mako==1.0.12 @@ -102,7 +102,7 @@ openstacksdk==0.17.0 os-client-config==1.32.0 os-service-types==1.7.0 osc-lib==1.12.1 -oslo.config==6.9.0 +oslo.config==6.10.0 oslo.context==2.22.1 oslo.i18n==3.23.1 oslo.log==3.44.0 @@ -115,7 +115,7 @@ paste==3.0.8 pastedeploy==2.0.1 pastescript==3.1.0 pathlib2==2.3.2 ; python_version < '3' -pbr==5.2.1 +pbr==5.3.0 prettytable==0.7.2 prov==1.5.1 psutil==5.6.3 From f0f02c7a8c6a340a60c473f116043b05eadb4f88 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 16 Jun 2019 22:24:28 +0200 Subject: [PATCH 88/88] Simplify unicodify Thanks Nicola! --- lib/galaxy/util/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index ff80f1df795..686946e9c96 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -1021,10 +1021,6 @@ def unicodify(value, encoding=DEFAULT_ENCODING, error='replace', strip_null=Fals """ if value is None: return value - if isinstance(value, text_type): - if strip_null: - return value.replace('\0', '') - return value try: if isinstance(value, bytearray): value = bytes(value)