Merge pull request #6647 from dannon/page_fixes

[18.05] Pages - fixes for encoded ids in the database

client/galaxy/scripts/galaxy.pages.js

@@ -444,10 +444,11 @@ WYMeditor.editor.prototype.dialog = function(dialogType, dialogFeatures, bodyHtm
                     if ($("#make-importable:checked").val() != null) make_importable = true;
 
                     grid.$("input[name=id]:checked").each(function() {
+                        var elt = $(this);
                         // Get item ID and name.
-                        var item_id = $(this).val();
+                        var item_id = elt.val();
                         // Use ':first' because there are many labels in table; the first one is the item name.
-                        var item_name = $(`label[for='${item_id}']:first`).text();
+                        var item_name = elt.closest('td').next('td').find('label').text();
 
                         if (make_importable) make_item_importable(item_info.controller, item_id, item_info.singular);
 

lib/galaxy/webapps/galaxy/controllers/page.py

@@ -72,6 +72,23 @@ _cp1252 = {
 }
 
 
+PAGE_MAXRAW = 10**15
+
+
+def _get_page_identifiers(item_id, app):
+    # Assume if item id is integer and less than 10**15, it's unencoded.
+    try:
+        decoded_id = int(item_id)
+        if decoded_id >= PAGE_MAXRAW:
+            raise ValueError("Identifier larger than maximum expected raw int, must be already encoded.")
+        encoded_id = app.security.encode_id(item_id)
+    except ValueError:
+        # It's an encoded id.
+        encoded_id = item_id
+        decoded_id = app.security.decode_id(item_id)
+    return (encoded_id, decoded_id)
+
+
 def format_bool(b):
     if b:
         return "yes"
@@ -620,7 +637,11 @@ class PageController(BaseUIController, SharableMixin,
         id = self.decode_id(id)
         page = trans.sa_session.query(model.Page).get(id)
         assert page.user == trans.user
-        return trans.fill_template("page/editor.mako", page=page)
+        content = page.latest_revision.content
+        processor = _PageContentProcessor(trans, _placeholderRenderForEdit)
+        processor.feed(content)
+        content = unicodify(processor.output(), 'utf-8')
+        return trans.fill_template("page/editor.mako", page=page, content=content)
 
     @web.expose
     @web.require_login("use Galaxy pages")
@@ -707,6 +728,10 @@ class PageController(BaseUIController, SharableMixin,
 
         # Sanitize content
         content = sanitize_html(content)
+        processor = _PageContentProcessor(trans, _placeholderRenderForSave)
+        processor.feed(content)
+        # Output is string, so convert to unicode for saving.
+        content = unicodify(processor.output(), 'utf-8')
 
         # Add a new revision to the page with the provided content.
         page_revision = model.PageRevision()
@@ -901,12 +926,10 @@ class PageController(BaseUIController, SharableMixin,
     def get_item(self, trans, id):
         return self.get_page(trans, id)
 
-    def _get_embedded_history_html(self, trans, id):
+    def _get_embedded_history_html(self, trans, decoded_id):
         """
         Returns html suitable for embedding in another page.
         """
-        # TODO: should be moved to history controller and/or called via ajax from the template
-        decoded_id = self.decode_id(id)
         # histories embedded in pages are set to importable when embedded, check for access here
         history = self.history_manager.get_accessible(decoded_id, trans.user, current_history=trans.history)
 
@@ -929,11 +952,11 @@ class PageController(BaseUIController, SharableMixin,
                                      content_dicts=contents)
         return filled
 
-    def _get_embedded_visualization_html(self, trans, id):
+    def _get_embedded_visualization_html(self, trans, encoded_id):
         """
         Returns html suitable for embedding visualizations in another page.
         """
-        visualization = self.get_visualization(trans, id, False, True)
+        visualization = self.get_visualization(trans, encoded_id, False, True)
         visualization.annotation = self.get_item_annotation_str(trans.sa_session, visualization.user, visualization)
         if not visualization:
             return None
@@ -955,11 +978,11 @@ class PageController(BaseUIController, SharableMixin,
     def _get_embed_html(self, trans, item_class, item_id):
         """ Returns HTML for embedding an item in a page. """
         item_class = self.get_class(item_class)
+        encoded_id, decoded_id = _get_page_identifiers(item_id, trans.app)
         if item_class == model.History:
-            return self._get_embedded_history_html(trans, item_id)
+            return self._get_embedded_history_html(trans, decoded_id)
 
         elif item_class == model.HistoryDatasetAssociation:
-            decoded_id = self.decode_id(item_id)
             dataset = self.hda_manager.get_accessible(decoded_id, trans.user)
             dataset = self.hda_manager.error_if_uploading(dataset)
 
@@ -969,14 +992,62 @@ class PageController(BaseUIController, SharableMixin,
                 return trans.fill_template("dataset/embed.mako", item=dataset, item_data=data)
 
         elif item_class == model.StoredWorkflow:
-            workflow = self.get_stored_workflow(trans, item_id, False, True)
+            workflow = self.get_stored_workflow(trans, encoded_id, False, True)
             workflow.annotation = self.get_item_annotation_str(trans.sa_session, workflow.user, workflow)
             if workflow:
                 self.get_stored_workflow_steps(trans, workflow)
                 return trans.fill_template("workflow/embed.mako", item=workflow, item_data=workflow.latest_workflow.steps)
 
         elif item_class == model.Visualization:
-            return self._get_embedded_visualization_html(trans, item_id)
+            return self._get_embedded_visualization_html(trans, encoded_id)
 
         elif item_class == model.Page:
             pass
+
+
+PLACEHOLDER_TEMPLATE = '''<div class="embedded-item {class_shorthand_lower} placeholder" id="{item_class}-{item_id}"><p class="title">Embedded Galaxy {class_shorthand} - '{item_name}'</p><p class="content">[Do not edit this block; Galaxy will fill it in with the annotated {class_shorthand} when it is displayed]</p></div>'''
+
+# This is a mapping of the id portion of page contents to the cssclass/shortname.
+PAGE_CLASS_MAPPING = {
+    'History': 'History',
+    'HistoryDatasetAssociation': 'Dataset',
+    'StoredWorkflow': 'Workflow',
+    'Visualization': 'Visualization'
+}
+
+
+def _placeholderRenderForEdit(trans, item_class, item_id):
+    return _placeholderRenderForSave(trans, item_class, item_id, encode=True)
+
+
+def _placeholderRenderForSave(trans, item_class, item_id, encode=False):
+    encoded_item_id, decoded_item_id = _get_page_identifiers(item_id, trans.app)
+    item_name = ''
+    if item_class == 'History':
+        history = trans.sa_session.query(trans.model.History).get(decoded_item_id)
+        history = managers.base.security_check(trans, history, False, True)
+        item_name = history.name
+    elif item_class == 'HistoryDatasetAssociation':
+        hda = trans.sa_session.query(trans.model.HistoryDatasetAssociation).get(decoded_item_id)
+        hda = managers.base.security_check(trans, hda , False, True)
+        item_name = hda.name
+    elif item_class == 'StoredWorkflow':
+        wf = trans.sa_session.query(trans.model.StoredWorkflow).get(decoded_item_id)
+        wf = managers.base.security_check(trans, wf, False, True)
+        item_name = wf.name
+    elif item_class == 'Visualization':
+        visualization = trans.sa_session.query(trans.model.Visualization).get(decoded_item_id)
+        visualization = managers.base.security_check(trans, visualization, False, True)
+        item_name = visualization.title
+    class_shorthand = PAGE_CLASS_MAPPING[item_class]
+    if encode:
+        item_id = encoded_item_id
+    else:
+        item_id = decoded_item_id
+    return PLACEHOLDER_TEMPLATE.format(
+        item_class=item_class,
+        class_shorthand=class_shorthand,
+        class_shorthand_lower=class_shorthand.lower(),
+        item_id=item_id,
+        item_name=item_name
+    )

scripts/pages_identifier_conversion.py

@@ -0,0 +1,57 @@
+import argparse
+import difflib
+import logging
+import os
+import sys
+
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
+
+import galaxy
+import galaxy.app
+import galaxy.config
+from galaxy.objectstore import build_object_store_from_config
+from galaxy.util import unicodify
+from galaxy.util.bunch import Bunch
+from galaxy.util.script import app_properties_from_args, populate_config_args
+from galaxy.web.security import SecurityHelper
+from galaxy.webapps.galaxy.controllers.page import _PageContentProcessor, _placeholderRenderForSave
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-k', '--secret-key', help='Key to convert pages with', default='')
+    parser.add_argument('-d', '--dry-run', help='No changes, just test it.', action='store_true')
+    populate_config_args(parser)
+    args = parser.parse_args()
+    properties = app_properties_from_args(args)
+    config = galaxy.config.Configuration(**properties)
+    secret = args.secret_key or config.id_secret
+    security_helper = SecurityHelper(id_secret=secret)
+    object_store = build_object_store_from_config(config)
+    if not config.database_connection:
+        print("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
+
+    model = galaxy.config.init_models_from_config(config, object_store=object_store)
+    session = model.context.current
+    pagerevs = session.query(model.PageRevision).all()
+    mock_trans = Bunch(app=Bunch(security=security_helper), model=model, user_is_admin=lambda: True, sa_session=session)
+    for p in pagerevs:
+        try:
+            processor = _PageContentProcessor(mock_trans, _placeholderRenderForSave)
+            processor.feed(p.content)
+            newcontent = unicodify(processor.output(), 'utf-8')
+            if p.content != newcontent:
+                if not args.dry_run:
+                    p.content = unicodify(processor.output(), 'utf-8')
+                    session.add(p)
+                    session.flush()
+                else:
+                    print("Modifying revision %s." % p.id)
+                    print(difflib.unified_diff(p.content, newcontent))
+        except Exception:
+            logging.exception("Error parsing page, rolling changes back and skipping revision %s.  Please report this error." % p.id)
+            session.rollback()
+
+
+if __name__ == '__main__':
+    main(sys.argv)

templates/webapps/galaxy/page/editor.mako

@@ -55,7 +55,7 @@
     </div>
 
     <div class="unified-panel-body">
-        <textarea name="page_content">${util.unicodify( page.latest_revision.content )}</textarea>
+        <textarea name="page_content">${ content }</textarea>
     </div>
 
 </%def>