From 8e6fbe34ca7c0dee4c6d08490f41139fa307365f Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 2 Mar 2015 16:33:43 -0500 Subject: [PATCH] tweak toolbox search to provide better (and more) results with the inclusion of toolhelp in index and with more versions of the same tools being massively installed the search results were inconsistent and inaccurate and incomplete. addresses https://trello.com/c/6Bvbduci/2486-toolbox-search-is-behaving-wrongly-differently --- client/galaxy/scripts/galaxy.workflows.js | 2 +- client/galaxy/scripts/mvc/tools.js | 2 +- lib/galaxy/tools/search/__init__.py | 44 +++++++++++++++-------- static/scripts/galaxy.workflows.js | 2 +- static/scripts/mvc/tools.js | 2 +- static/scripts/packed/galaxy.workflows.js | 2 +- static/scripts/packed/mvc/tools.js | 2 +- 7 files changed, 35 insertions(+), 21 deletions(-) diff --git a/client/galaxy/scripts/galaxy.workflows.js b/client/galaxy/scripts/galaxy.workflows.js index 3bbd302ffc2..652a3bbbc6a 100644 --- a/client/galaxy/scripts/galaxy.workflows.js +++ b/client/galaxy/scripts/galaxy.workflows.js @@ -24,7 +24,7 @@ $( function() { $(this).addClass("search_active"); // input.addClass(config.loadingClass); // Add '*' to facilitate partial matching. - var q = this.value + '*'; + var q = this.value; // Stop previous ajax-request if (this.timer) { clearTimeout(this.timer); diff --git a/client/galaxy/scripts/mvc/tools.js b/client/galaxy/scripts/mvc/tools.js index 0fd3f98c0a8..02f834de51a 100644 --- a/client/galaxy/scripts/mvc/tools.js +++ b/client/galaxy/scripts/mvc/tools.js @@ -376,7 +376,7 @@ var ToolSearch = Backbone.Model.extend({ } // Do search via AJAX. - var q = query + '*'; + var q = query; // Stop previous ajax-request if (this.timer) { clearTimeout(this.timer); diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 9c94de83b73..ad27285a42e 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -1,18 +1,27 @@ -from galaxy.eggs import require +""" +Module for building and searching the index of tools +installed within this Galaxy. +""" +from galaxy import eggs from galaxy.web.framework.helpers import to_unicode -require( "Whoosh" ) +eggs.require( "Whoosh" ) from whoosh.filedb.filestore import RamStorage from whoosh.fields import Schema, STORED, TEXT from whoosh.scoring import BM25F from whoosh.qparser import MultifieldParser -schema = Schema( id=STORED, title=TEXT, description=TEXT, help=TEXT ) - +schema = Schema( id=STORED, + title=TEXT, + description=TEXT, + section=TEXT, + help=TEXT ) +import logging +log = logging.getLogger( __name__ ) class ToolBoxSearch( object ): """ Support searching tools in a toolbox. This implementation uses - the "whoosh" search library. + the Whoosh search library. """ def __init__( self, toolbox, index_help=True ): @@ -26,13 +35,13 @@ class ToolBoxSearch( object ): self.storage = RamStorage() self.index = self.storage.create_index( schema ) writer = self.index.writer() - ## TODO: would also be nice to search section headers. for id, tool in self.toolbox.tools(): add_doc_kwds = { "id": id, - "title": to_unicode(tool.name), - "description": to_unicode(tool.description), - "help": to_unicode(""), + "title": to_unicode( tool.name ), + "description": to_unicode( tool.description ), + "section": to_unicode( tool.get_panel_section()[1] if len( tool.get_panel_section() ) == 2 else '' ), + "help": to_unicode( "" ), } if index_help and tool.help: try: @@ -45,13 +54,18 @@ class ToolBoxSearch( object ): writer.commit() def search( self, query, return_attribute='id' ): - # Change field boosts for searcher to place more weight on title, description than help. + # Change field boosts for searcher searcher = self.index.searcher( weighting=BM25F( - field_B={ 'title_B': 3, 'description_B': 2, 'help_B': 1 } + field_B={ 'title_B': 9, + 'section_B': 3, + 'description_B': 2, + 'help_B': 0.5 } ) ) - # Set query to search title, description, and help. - parser = MultifieldParser( [ 'title', 'description', 'help' ], schema=schema ) - results = searcher.search( parser.parse( query ) ) - return [ result[ return_attribute ] for result in results ] + # Set query to search title, description, section, and help. + parser = MultifieldParser( [ 'title', 'description', 'section', 'help' ], schema=schema ) + # Perform the search + hits = searcher.search( parser.parse( '*' + query + '*' ), limit=20 ) + + return [ hit[ return_attribute ] for hit in hits ] diff --git a/static/scripts/galaxy.workflows.js b/static/scripts/galaxy.workflows.js index 3bbd302ffc2..652a3bbbc6a 100644 --- a/static/scripts/galaxy.workflows.js +++ b/static/scripts/galaxy.workflows.js @@ -24,7 +24,7 @@ $( function() { $(this).addClass("search_active"); // input.addClass(config.loadingClass); // Add '*' to facilitate partial matching. - var q = this.value + '*'; + var q = this.value; // Stop previous ajax-request if (this.timer) { clearTimeout(this.timer); diff --git a/static/scripts/mvc/tools.js b/static/scripts/mvc/tools.js index 0fd3f98c0a8..02f834de51a 100644 --- a/static/scripts/mvc/tools.js +++ b/static/scripts/mvc/tools.js @@ -376,7 +376,7 @@ var ToolSearch = Backbone.Model.extend({ } // Do search via AJAX. - var q = query + '*'; + var q = query; // Stop previous ajax-request if (this.timer) { clearTimeout(this.timer); diff --git a/static/scripts/packed/galaxy.workflows.js b/static/scripts/packed/galaxy.workflows.js index 0d4ce45c572..c0506961d4b 100644 --- a/static/scripts/packed/galaxy.workflows.js +++ b/static/scripts/packed/galaxy.workflows.js @@ -1 +1 @@ -$(function(){if(window.lt_ie_7){show_modal("Browser not supported","Sorry, the workflow editor is not supported for IE6 and below.");return}$("#tool-search-query").click(function(){$(this).focus();$(this).select()}).keyup(function(){$(this).css("font-style","normal");if(this.value.length<3){reset_tool_search(false)}else{if(this.value!=this.lastValue){$(this).addClass("search_active");var g=this.value+"*";if(this.timer){clearTimeout(this.timer)}$("#search-spinner").show();this.timer=setTimeout(function(){$.get(tool_search_url,{query:g},function(i){$("#search-no-results").hide();$(".toolSectionWrapper").hide();$(".toolSectionWrapper").find(".toolTitle").hide();if(i.length!=0){var h=$.map(i,function(k,j){return"link-"+k});$(h).each(function(j,k){$("[id='"+k+"']").parent().addClass("search_match");$("[id='"+k+"']").parent().show().parent().parent().show().parent().show()});$(".toolPanelLabel").each(function(){var l=$(this);var k=l.next();var j=true;while(k.length!==0&&k.hasClass("toolTitle")){if(k.is(":visible")){j=false;break}else{k=k.next()}}if(j){l.hide()}})}else{$("#search-no-results").show()}$("#search-spinner").hide()},"json")},200)}}this.lastValue=this.value});canvas_manager=new CanvasManager($("#canvas-viewport"),$("#overview"));reset();$.ajax({url:get_datatypes_url,dataType:"json",cache:false,success:function(g){populate_datatype_info(g);$.ajax({url:load_workflow_url,data:{id:workflow_id,_:"true"},dataType:"json",cache:false,success:function(h){reset();workflow.from_simple(h);workflow.has_changes=false;workflow.fit_canvas_to_nodes();scroll_to_nodes();canvas_manager.draw_overview();upgrade_message="";$.each(h.upgrade_messages,function(j,i){upgrade_message+=("
  • Step "+(parseInt(j,10)+1)+": "+workflow.nodes[j].name+"
    • "});if(upgrade_message){show_modal("Workflow loaded with changes","Problems were encountered loading this workflow (possibly a result of tool upgrades). Please review the following parameters and then save.",{Continue:hide_modal})}else{hide_modal()}show_workflow_parameters()},beforeSubmit:function(h){show_message("Loading workflow","progress")}})}});$(document).ajaxStart(function(){active_ajax_call=true;$(document).bind("ajaxStop.global",function(){active_ajax_call=false})});$(document).ajaxError(function(i,g){var h=g.responseText||g.statusText||"Could not connect to server";show_modal("Server error",h,{"Ignore error":hide_modal});return false});make_popupmenu($("#workflow-options-button"),{Save:save_current_workflow,Run:function(){window.location=run_workflow_url},"Edit Attributes":c,"Auto Re-layout":f,Close:close_editor});function b(){workflow.clear_active_node();$(".right-content").hide();var j="";for(var h in workflow.nodes){var i=workflow.nodes[h];if(i.type=="tool"){j+="
    Step "+i.id+" - "+i.name+"
    ";for(var k in i.output_terminals){var g=i.output_terminals[k];if($.inArray(g.name,i.workflow_outputs)!=-1){j+="

    "+g.name+"

    "}else{j+="

    "+g.name+"

    "}}j+="
    "}}$("#output-fill-area").html(j);$("#output-fill-area input").bind("click",function(){var n=this.name.split("|")[0];var l=this.name.split("|")[1];if(this.checked){if($.inArray(l,workflow.nodes[n].workflow_outputs)==-1){workflow.nodes[n].workflow_outputs.push(l)}}else{while($.inArray(l,workflow.nodes[n].workflow_outputs)!=-1){var m=$.inArray(l,workflow.nodes[n].workflow_outputs);workflow.nodes[n].workflow_outputs=workflow.nodes[n].workflow_outputs.slice(0,m).concat(workflow.nodes[n].workflow_outputs.slice(m+1))}}workflow.has_changes=true});$("#workflow-output-area").show()}function f(){workflow.layout();workflow.fit_canvas_to_nodes();scroll_to_nodes();canvas_manager.draw_overview()}function c(){workflow.clear_active_node();$(".right-content").hide();$("#edit-attributes").show()}overview_size=$.jStorage.get("overview-size");if(overview_size!==undefined){$("#overview-border").css({width:overview_size,height:overview_size})}if($.jStorage.get("overview-off")){d()}else{e()}$("#overview-border").bind("dragend",function(h,j){var k=$(this).offsetParent();var i=k.offset();var g=Math.max(k.width()-(j.offsetX-i.left),k.height()-(j.offsetY-i.top));$.jStorage.set("overview-size",g+"px")});function e(){$.jStorage.set("overview-off",false);$("#overview-border").css("right","0px");$("#close-viewport").css("background-position","0px 0px")}function d(){$.jStorage.set("overview-off",true);$("#overview-border").css("right","20000px");$("#close-viewport").css("background-position","12px 0px")}$("#close-viewport").click(function(){if($("#overview-border").css("right")==="0px"){d()}else{e()}});window.onbeforeunload=function(){if(workflow&&workflow.has_changes){return"There are unsaved changes to your workflow which will be lost."}};$("div.toolSectionBody").hide();$("div.toolSectionTitle > span").wrap("");var a=null;$("div.toolSectionTitle").each(function(){var g=$(this).next("div.toolSectionBody");$(this).click(function(){if(g.is(":hidden")){if(a){a.slideUp("fast")}a=g;g.slideDown("fast")}else{g.slideUp("fast");a=null}})});async_save_text("workflow-name","workflow-name",rename_async_url,"new_name");$("#workflow-tag").click(function(){$(".tag-area").click();return false});async_save_text("workflow-annotation","workflow-annotation",annotate_async_url,"new_annotation",25,true,4)});function reset(){if(workflow){workflow.remove_all()}workflow=new Workflow($("#canvas-container"))}function scroll_to_nodes(){var a=$("#canvas-viewport");var d=$("#canvas-container");var c,b;if(d.width().toolForm:last>.toolFormTitle>.buttons").click(function(){action_to_rem=$(this).closest(".toolForm",".action_tag").children(".action_tag:first").text();$(this).closest(".toolForm").remove();delete workflow.active_node.post_job_actions[action_to_rem];workflow.active_form_has_changes=true})}function display_pja_list(){return pja_list}function display_file_list(b){addlist="";return addlist}function new_pja(d,b,a){if(a.post_job_actions===undefined){a.post_job_actions={}}if(a.post_job_actions[d+b]===undefined){var c={};c.action_type=d;c.output_name=b;a.post_job_actions[d+b]=null;a.post_job_actions[d+b]=c;display_pja(c,a);workflow.active_form_has_changes=true;return true}else{return false}}function show_workflow_parameters(){var c=/\$\{.+?\}/g;var b=[];var f=$("#workflow-parameters-container");var e=$("#workflow-parameters-box");var a="";var d=[];$.each(workflow.nodes,function(g,h){var i=h.form_html.match(c);if(i){d=d.concat(i)}if(h.post_job_actions){$.each(h.post_job_actions,function(j,l){if(l.action_arguments){$.each(l.action_arguments,function(m,n){var o=n.match(c);if(o){d=d.concat(o)}})}});if(d){$.each(d,function(j,l){if($.inArray(l,b)===-1){b.push(l)}})}}});if(b&&b.length!==0){$.each(b,function(g,h){a+="
    "+h.substring(2,h.length-1)+"
    "});f.html(a);e.show()}else{f.html(a);e.hide()}}function show_form_for_tool(c,b){$(".right-content").hide();$("#right-content").show().html(c);if(b){$("#right-content").find(".toolForm:first").after("

    Edit Step Attributes
    Add an annotation or notes to this step; annotations are available when a workflow is viewed.
    ")}if(b&&b.type=="tool"){pjastr="

    Edit Step Actions
    "+display_pja_list()+"
    "+display_file_list(b)+"
    Create
    ";pjastr+="
    Add actions to this step; actions are applied when this workflow step completes.
    ";$("#right-content").find(".toolForm").after(pjastr);for(var a in b.post_job_actions){if(a!="undefined"){display_pja(b.post_job_actions[a],b)}}$("#add_pja").click(function(){new_pja($("#new_pja_list").val(),$("#node_data_list").val(),b)})}$("#right-content").find("form").ajaxForm({type:"POST",dataType:"json",success:function(d){workflow.active_form_has_changes=false;b.update_field_data(d);show_workflow_parameters()},beforeSubmit:function(d){d.push({name:"tool_state",value:b.tool_state});d.push({name:"_",value:"true"})}}).each(function(){var d=this;$(this).find("select[refresh_on_change='true']").change(function(){$(d).submit()});$(this).find("input[refresh_on_change='true']").change(function(){$(d).submit()});$(this).find(".popupmenu").each(function(){var g=$(this).parents("div.form-row").attr("id");var e=$('▼');var f={};$(this).find("button").each(function(){var h=$(this).attr("name");var i=$(this).attr("value");f[$(this).text()]=function(){$(d).append("").submit()}});e.insertAfter(this);$(this).remove();make_popupmenu(e,f)});$(this).find("input,textarea,select").each(function(){$(this).bind("focus click",function(){workflow.active_form_has_changes=true})})})}var close_editor=function(){workflow.check_changes_in_active_form();if(workflow&&workflow.has_changes){do_close=function(){window.onbeforeunload=undefined;window.document.location=workflow_index_url};show_modal("Close workflow editor","There are unsaved changes to your workflow which will be lost.",{Cancel:hide_modal,"Save Changes":function(){save_current_workflow(null,do_close)}},{"Don't Save":do_close})}else{window.document.location=workflow_index_url}};var save_current_workflow=function(b,c){show_message("Saving workflow","progress");workflow.check_changes_in_active_form();if(!workflow.has_changes){hide_modal();if(c){c()}return}workflow.rectify_workflow_outputs();var a=function(d){$.ajax({url:save_workflow_url,type:"POST",data:{id:workflow_id,workflow_data:function(){return JSON.stringify(workflow.to_simple())},_:"true"},dataType:"json",success:function(g){var e=$("
    ").text(g.message);if(g.errors){e.addClass("warningmark");var f=$("