opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

make field boosts for TS search configurable

Browse Source
This commit is contained in:
Martin Cech
2015-06-19 17:25:53 -04:00
parent eb54b1d854
commit 1bb4e99e27
6 changed files with 83 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/tool_shed.ini.sample
+18
View File
@@ -59,6 +59,24 @@ session_secret = changethisinproduction
#toolshed_search_on = True
#whoosh_index_dir = database/toolshed_whoosh_indexes
# The following boosts are used to customize this instance's TS search.
# The higher the boost, the more importance the scoring algorithm gives to the
# given field.
# For searching repositories at /api/repositories:
#repo_name_boost = 0.9
#repo_description_boost = 0.6
#repo_long_description_boost = 0.5
#repo_homepage_url_boost = 0.3
#repo_remote_repository_url_boost = 0.2
#repo_owner_username_boost = 0.3
# For searching tools at /api/tools
#tool_name_boost = 1.2
#tool_description_boost = 0.6
#tool_help_boost = 0.4
#tool_repo_owner_username = 0.3
# -- Analytics
# You can enter tracking code here to track visitor's behavior
lib/galaxy/webapps/tool_shed/api/repositories.py
+22 -4
View File
@@ -5,7 +5,7 @@ import tarfile
import StringIO
import tempfile
from time import strftime
from collections import namedtuple
from cgi import FieldStorage
from galaxy import util
@@ -391,16 +391,34 @@ class RepositoriesController( BaseAPIController ):
Also TS config option toolshed_search_on has to be True and
whoosh_index_dir has to be specified.
"""
if not self.app.config.toolshed_search_on:
conf = self.app.config
if not conf.toolshed_search_on:
raise ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' )
if not self.app.config.whoosh_index_dir:
if not conf.whoosh_index_dir:
raise ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please contact the administrator.' )
search_term = q.strip()
if len( search_term ) < 3:
raise RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' )
repo_search = RepoSearch()
results = repo_search.search( trans, search_term, page )
Boosts = namedtuple( 'Boosts', [ 'repo_name_boost',
'repo_description_boost',
'repo_long_description_boost',
'repo_homepage_url_boost',
'repo_remote_repository_url_boost',
'repo_owner_username_boost' ] )
boosts = Boosts( float( conf.get( 'repo_name_boost', 0.9 ) ),
float( conf.get( 'repo_description_boost', 0.6 ) ),
float( conf.get( 'repo_long_description_boost', 0.5 ) ),
float( conf.get( 'repo_homepage_url_boost', 0.3 ) ),
float( conf.get( 'repo_remote_repository_url_boost', 0.2 ) ),
float( conf.get( 'repo_owner_username_boost', 0.3 ) ) )
results = repo_search.search( trans,
search_term,
page,
boosts )
results[ 'hostname' ] = web.url_for( '/', qualified = True )
return results
lib/galaxy/webapps/tool_shed/api/tools.py
+19 -4
View File
@@ -1,6 +1,7 @@
import json
import logging
import os
from collections import namedtuple
from galaxy import web
from galaxy import util
from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
@@ -46,7 +47,7 @@ class ToolsController( BaseAPIController ):
else:
page = kwd.get( 'page', 1 )
return_jsonp = util.asbool( kwd.get( 'jsonp', False ) )
callback = kwd.get( 'callback', 'callback' )
callback = kwd.get( 'callback', 'callback' )
search_results = self._search( trans, q, page )
if return_jsonp:
response = str( '%s(%s);' % ( callback, json.dumps( search_results ) ) )
@@ -62,15 +63,29 @@ class ToolsController( BaseAPIController ):
Also TS config option toolshed_search_on has to be True and
whoosh_index_dir has to be specified.
"""
if not self.app.config.toolshed_search_on:
conf = self.app.config
if not conf.toolshed_search_on:
raise ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' )
if not self.app.config.whoosh_index_dir:
if not conf.whoosh_index_dir:
raise ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please contact the administrator.' )
search_term = q.strip()
if len( search_term ) < 3:
raise RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' )
tool_search = ToolSearch()
results = tool_search.search( trans, search_term, page )
Boosts = namedtuple( 'Boosts', [ 'tool_name_boost',
'tool_description_boost',
'tool_help_boost',
'tool_repo_owner_username_boost' ] )
boosts = Boosts( float( conf.get( 'tool_name_boost', 1.2 ) ),
float( conf.get( 'tool_description_boost', 0.6 ) ),
float( conf.get( 'tool_help_boost', 0.4 ) ),
float( conf.get( 'tool_repo_owner_username_boost', 0.3 ) ) )
results = tool_search.search( trans,
search_term,
page,
boosts )
results[ 'hostname' ] = web.url_for( '/', qualified = True )
return results
lib/galaxy/webapps/tool_shed/config.py
+11 -1
View File
@@ -42,9 +42,19 @@ class Configuration( object ):
self.database_connection = kwargs.get( "database_connection", False )
self.database_engine_options = get_database_engine_options( kwargs )
self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) )
# Whoosh search
# Repository and Tool search API
self.toolshed_search_on = string_as_bool( kwargs.get( "toolshed_search_on", True ) )
self.whoosh_index_dir = kwargs.get( "whoosh_index_dir", 'database/toolshed_whoosh_indexes' )
self.repo_name_boost = kwargs.get( "repo_name_boost", 0.9 )
self.repo_description_boost = kwargs.get( "repo_description_boost", 0.6 )
self.repo_long_description_boost = kwargs.get( "repo_long_description_boost", 0.5 )
self.repo_homepage_url_boost = kwargs.get( "repo_homepage_url_boost", 0.3 )
self.repo_remote_repository_url_boost = kwargs.get( "repo_remote_repository_url_boost", 0.2 )
self.repo_owner_username_boost = kwargs.get( "repo_owner_username_boost", 0.3 )
self.tool_name_boost = kwargs.get( "tool_name_boost", 1.2 )
self.tool_description_boost = kwargs.get( "tool_description_boost", 0.6 )
self.tool_help_boost = kwargs.get( "tool_help_boost", 0.4 )
self.tool_repo_owner_username = kwargs.get( "tool_repo_owner_username", 0.3 )
# Analytics
self.ga_code = kwargs.get( "ga_code", None )
self.session_duration = int(kwargs.get( 'session_duration', 0 ))
lib/galaxy/webapps/tool_shed/search/repo_search.py
+8 -7
View File
@@ -64,11 +64,12 @@ class RepoWeighting( scoring.BM25F ):
class RepoSearch( object ):
def search( self, trans, search_term, page, **kwd ):
def search( self, trans, search_term, page, boosts ):
"""
Perform the search on the given search_term
:param search_term: unicode encoded string with the search term(s)
:param boosts: namedtuple containing custom boosts for searchfields, see api/repositories.py
:returns results: dictionary containing number of hits, hits themselves and matched terms for each
"""
@@ -81,12 +82,12 @@ class RepoSearch( object ):
# http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf
# http://en.wikipedia.org/wiki/Okapi_BM25
# __Basically__ the higher number the bigger weight.
repo_weighting = RepoWeighting( field_B = { 'name_B' : 0.9,
'description_B' : 0.6,
'long_description_B' : 0.5,
'homepage_url_B' : 0.3,
'remote_repository_url_B' : 0.2,
'repo_owner_username' : 0.3 } )
repo_weighting = RepoWeighting( field_B = { 'name_B' : boosts.repo_name_boost,
'description_B' : boosts.repo_description_boost,
'long_description_B' : boosts.repo_long_description_boost,
'homepage_url_B' : boosts.repo_homepage_url_boost,
'remote_repository_url_B' : boosts.repo_remote_repository_url_boost,
'repo_owner_username' : boosts.repo_owner_username_boost } )
searcher = index.searcher( weighting = repo_weighting )
lib/galaxy/webapps/tool_shed/search/tool_search.py
+5 -5
View File
@@ -30,7 +30,7 @@ tool_schema = Schema(
class ToolSearch( object ):
def search( self, trans, search_term, page, **kwd ):
def search( self, trans, search_term, page, boosts ):
"""
Perform the search on the given search_term
@@ -47,10 +47,10 @@ class ToolSearch( object ):
# http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf
# http://en.wikipedia.org/wiki/Okapi_BM25
# __Basically__ the higher number the bigger weight.
tool_weighting = scoring.BM25F( field_B = { 'name_B' : 0.9,
'description_B' : 0.6,
'help_B' : 0.4,
'repo_owner_username_B' : 0.3 } )
tool_weighting = scoring.BM25F( field_B = { 'name_B' : boosts.tool_name_boost,
'description_B' : boosts.tool_description_boost,
'help_B' : boosts.tool_help_boost,
'repo_owner_username_B' : boosts.tool_repo_owner_username_boost } )
searcher = index.searcher( weighting = tool_weighting )
parser = MultifieldParser( [