From 1bb4e99e27774b04f1485128949c489838b088e1 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 19 Jun 2015 17:25:53 -0400 Subject: [PATCH] make field boosts for TS search configurable --- config/tool_shed.ini.sample | 18 +++++++++++++ .../webapps/tool_shed/api/repositories.py | 26 ++++++++++++++++--- lib/galaxy/webapps/tool_shed/api/tools.py | 23 +++++++++++++--- lib/galaxy/webapps/tool_shed/config.py | 12 ++++++++- .../webapps/tool_shed/search/repo_search.py | 15 ++++++----- .../webapps/tool_shed/search/tool_search.py | 10 +++---- 6 files changed, 83 insertions(+), 21 deletions(-) diff --git a/config/tool_shed.ini.sample b/config/tool_shed.ini.sample index ca9c15fe99d..58c0e3cb0e4 100644 --- a/config/tool_shed.ini.sample +++ b/config/tool_shed.ini.sample @@ -59,6 +59,24 @@ session_secret = changethisinproduction #toolshed_search_on = True #whoosh_index_dir = database/toolshed_whoosh_indexes +# The following boosts are used to customize this instance's TS search. +# The higher the boost, the more importance the scoring algorithm gives to the +# given field. + +# For searching repositories at /api/repositories: +#repo_name_boost = 0.9 +#repo_description_boost = 0.6 +#repo_long_description_boost = 0.5 +#repo_homepage_url_boost = 0.3 +#repo_remote_repository_url_boost = 0.2 +#repo_owner_username_boost = 0.3 + +# For searching tools at /api/tools +#tool_name_boost = 1.2 +#tool_description_boost = 0.6 +#tool_help_boost = 0.4 +#tool_repo_owner_username = 0.3 + # -- Analytics # You can enter tracking code here to track visitor's behavior diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index c5eca618cb4..3424cd2ac4f 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -5,7 +5,7 @@ import tarfile import StringIO import tempfile from time import strftime - +from collections import namedtuple from cgi import FieldStorage from galaxy import util @@ -391,16 +391,34 @@ class RepositoriesController( BaseAPIController ): Also TS config option toolshed_search_on has to be True and whoosh_index_dir has to be specified. """ - if not self.app.config.toolshed_search_on: + conf = self.app.config + if not conf.toolshed_search_on: raise ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' ) - if not self.app.config.whoosh_index_dir: + if not conf.whoosh_index_dir: raise ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please contact the administrator.' ) search_term = q.strip() if len( search_term ) < 3: raise RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' ) repo_search = RepoSearch() - results = repo_search.search( trans, search_term, page ) + + Boosts = namedtuple( 'Boosts', [ 'repo_name_boost', + 'repo_description_boost', + 'repo_long_description_boost', + 'repo_homepage_url_boost', + 'repo_remote_repository_url_boost', + 'repo_owner_username_boost' ] ) + boosts = Boosts( float( conf.get( 'repo_name_boost', 0.9 ) ), + float( conf.get( 'repo_description_boost', 0.6 ) ), + float( conf.get( 'repo_long_description_boost', 0.5 ) ), + float( conf.get( 'repo_homepage_url_boost', 0.3 ) ), + float( conf.get( 'repo_remote_repository_url_boost', 0.2 ) ), + float( conf.get( 'repo_owner_username_boost', 0.3 ) ) ) + + results = repo_search.search( trans, + search_term, + page, + boosts ) results[ 'hostname' ] = web.url_for( '/', qualified = True ) return results diff --git a/lib/galaxy/webapps/tool_shed/api/tools.py b/lib/galaxy/webapps/tool_shed/api/tools.py index 2d014e30d30..d2109221f27 100644 --- a/lib/galaxy/webapps/tool_shed/api/tools.py +++ b/lib/galaxy/webapps/tool_shed/api/tools.py @@ -1,6 +1,7 @@ import json import logging import os +from collections import namedtuple from galaxy import web from galaxy import util from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless @@ -46,7 +47,7 @@ class ToolsController( BaseAPIController ): else: page = kwd.get( 'page', 1 ) return_jsonp = util.asbool( kwd.get( 'jsonp', False ) ) - callback = kwd.get( 'callback', 'callback' ) + callback = kwd.get( 'callback', 'callback' ) search_results = self._search( trans, q, page ) if return_jsonp: response = str( '%s(%s);' % ( callback, json.dumps( search_results ) ) ) @@ -62,15 +63,29 @@ class ToolsController( BaseAPIController ): Also TS config option toolshed_search_on has to be True and whoosh_index_dir has to be specified. """ - if not self.app.config.toolshed_search_on: + conf = self.app.config + if not conf.toolshed_search_on: raise ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' ) - if not self.app.config.whoosh_index_dir: + if not conf.whoosh_index_dir: raise ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please contact the administrator.' ) search_term = q.strip() if len( search_term ) < 3: raise RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' ) tool_search = ToolSearch() - results = tool_search.search( trans, search_term, page ) + + Boosts = namedtuple( 'Boosts', [ 'tool_name_boost', + 'tool_description_boost', + 'tool_help_boost', + 'tool_repo_owner_username_boost' ] ) + boosts = Boosts( float( conf.get( 'tool_name_boost', 1.2 ) ), + float( conf.get( 'tool_description_boost', 0.6 ) ), + float( conf.get( 'tool_help_boost', 0.4 ) ), + float( conf.get( 'tool_repo_owner_username_boost', 0.3 ) ) ) + + results = tool_search.search( trans, + search_term, + page, + boosts ) results[ 'hostname' ] = web.url_for( '/', qualified = True ) return results diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index 0e967cada0e..da796d28e27 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -42,9 +42,19 @@ class Configuration( object ): self.database_connection = kwargs.get( "database_connection", False ) self.database_engine_options = get_database_engine_options( kwargs ) self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) ) - # Whoosh search + # Repository and Tool search API self.toolshed_search_on = string_as_bool( kwargs.get( "toolshed_search_on", True ) ) self.whoosh_index_dir = kwargs.get( "whoosh_index_dir", 'database/toolshed_whoosh_indexes' ) + self.repo_name_boost = kwargs.get( "repo_name_boost", 0.9 ) + self.repo_description_boost = kwargs.get( "repo_description_boost", 0.6 ) + self.repo_long_description_boost = kwargs.get( "repo_long_description_boost", 0.5 ) + self.repo_homepage_url_boost = kwargs.get( "repo_homepage_url_boost", 0.3 ) + self.repo_remote_repository_url_boost = kwargs.get( "repo_remote_repository_url_boost", 0.2 ) + self.repo_owner_username_boost = kwargs.get( "repo_owner_username_boost", 0.3 ) + self.tool_name_boost = kwargs.get( "tool_name_boost", 1.2 ) + self.tool_description_boost = kwargs.get( "tool_description_boost", 0.6 ) + self.tool_help_boost = kwargs.get( "tool_help_boost", 0.4 ) + self.tool_repo_owner_username = kwargs.get( "tool_repo_owner_username", 0.3 ) # Analytics self.ga_code = kwargs.get( "ga_code", None ) self.session_duration = int(kwargs.get( 'session_duration', 0 )) diff --git a/lib/galaxy/webapps/tool_shed/search/repo_search.py b/lib/galaxy/webapps/tool_shed/search/repo_search.py index 1af111a1028..a6a295e5dc6 100644 --- a/lib/galaxy/webapps/tool_shed/search/repo_search.py +++ b/lib/galaxy/webapps/tool_shed/search/repo_search.py @@ -64,11 +64,12 @@ class RepoWeighting( scoring.BM25F ): class RepoSearch( object ): - def search( self, trans, search_term, page, **kwd ): + def search( self, trans, search_term, page, boosts ): """ Perform the search on the given search_term :param search_term: unicode encoded string with the search term(s) + :param boosts: namedtuple containing custom boosts for searchfields, see api/repositories.py :returns results: dictionary containing number of hits, hits themselves and matched terms for each """ @@ -81,12 +82,12 @@ class RepoSearch( object ): # http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf # http://en.wikipedia.org/wiki/Okapi_BM25 # __Basically__ the higher number the bigger weight. - repo_weighting = RepoWeighting( field_B = { 'name_B' : 0.9, - 'description_B' : 0.6, - 'long_description_B' : 0.5, - 'homepage_url_B' : 0.3, - 'remote_repository_url_B' : 0.2, - 'repo_owner_username' : 0.3 } ) + repo_weighting = RepoWeighting( field_B = { 'name_B' : boosts.repo_name_boost, + 'description_B' : boosts.repo_description_boost, + 'long_description_B' : boosts.repo_long_description_boost, + 'homepage_url_B' : boosts.repo_homepage_url_boost, + 'remote_repository_url_B' : boosts.repo_remote_repository_url_boost, + 'repo_owner_username' : boosts.repo_owner_username_boost } ) searcher = index.searcher( weighting = repo_weighting ) diff --git a/lib/galaxy/webapps/tool_shed/search/tool_search.py b/lib/galaxy/webapps/tool_shed/search/tool_search.py index de345f30739..c829932b6ae 100644 --- a/lib/galaxy/webapps/tool_shed/search/tool_search.py +++ b/lib/galaxy/webapps/tool_shed/search/tool_search.py @@ -30,7 +30,7 @@ tool_schema = Schema( class ToolSearch( object ): - def search( self, trans, search_term, page, **kwd ): + def search( self, trans, search_term, page, boosts ): """ Perform the search on the given search_term @@ -47,10 +47,10 @@ class ToolSearch( object ): # http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf # http://en.wikipedia.org/wiki/Okapi_BM25 # __Basically__ the higher number the bigger weight. - tool_weighting = scoring.BM25F( field_B = { 'name_B' : 0.9, - 'description_B' : 0.6, - 'help_B' : 0.4, - 'repo_owner_username_B' : 0.3 } ) + tool_weighting = scoring.BM25F( field_B = { 'name_B' : boosts.tool_name_boost, + 'description_B' : boosts.tool_description_boost, + 'help_B' : boosts.tool_help_boost, + 'repo_owner_username_B' : boosts.tool_repo_owner_username_boost } ) searcher = index.searcher( weighting = tool_weighting ) parser = MultifieldParser( [