mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-25 00:12:08 +08:00
Merge pull request #334 from martenson/ts-search-expansion
TS search refactoring and bugfix
This commit is contained in:
John Chilton
@@ -16,13 +16,16 @@ from galaxy.exceptions import InsufficientPermissionsException
|
||||
from galaxy.exceptions import ActionInputError
|
||||
from galaxy.exceptions import ObjectNotFound
|
||||
from galaxy.exceptions import MalformedId
|
||||
from galaxy.exceptions import ConfigDoesNotAllowException
|
||||
from galaxy.datatypes import checkers
|
||||
from galaxy.model.orm import and_
|
||||
from galaxy.web import _future_expose_api as expose_api
|
||||
from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless
|
||||
from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
|
||||
from galaxy.web.base.controller import BaseAPIController
|
||||
from galaxy.web.base.controller import HTTPBadRequest
|
||||
from galaxy.web.framework.helpers import time_ago
|
||||
from galaxy.webapps.tool_shed.search.repo_search import RepoSearch
|
||||
|
||||
from tool_shed.capsule import capsule_manager
|
||||
from tool_shed.metadata import repository_metadata_manager
|
||||
@@ -316,20 +319,54 @@ class RepositoriesController( BaseAPIController ):
|
||||
return_dict[ key ] = val
|
||||
return return_dict
|
||||
|
||||
@web.expose_api_anonymous
|
||||
@expose_api_raw_anonymous_and_sessionless
|
||||
def index( self, trans, deleted=False, owner=None, name=None, **kwd ):
|
||||
"""
|
||||
GET /api/repositories
|
||||
Displays a collection of repositories with optional criteria.
|
||||
|
||||
:param deleted: True/False, displays repositories that are or are not set to deleted.
|
||||
:param owner: the owner's public username.
|
||||
:param name: the repository name.
|
||||
:param q: (optional)if present search on the given query will be performed
|
||||
:type q: str
|
||||
|
||||
Displays a collection (list) of repositories.
|
||||
:param page: (optional)requested page of the search
|
||||
:type page: int
|
||||
|
||||
:param jsonp: (optional)flag whether to use jsonp format response, defaults to False
|
||||
:type jsonp: bool
|
||||
|
||||
:param callback: (optional)name of the function to wrap callback in
|
||||
used only when jsonp is true, defaults to 'callback'
|
||||
:type callback: str
|
||||
|
||||
:param deleted: (optional)displays repositories that are or are not set to deleted.
|
||||
:type deleted: bool
|
||||
|
||||
:param owner: (optional)the owner's public username.
|
||||
:type owner: str
|
||||
|
||||
:param name: (optional)the repository name.
|
||||
:type name: str
|
||||
|
||||
:returns dict: object containing list of results
|
||||
|
||||
Examples:
|
||||
GET http://localhost:9009/api/repositories
|
||||
GET http://localhost:9009/api/repositories?q=fastq
|
||||
"""
|
||||
# Example URL: http://localhost:9009/api/repositories
|
||||
repository_dicts = []
|
||||
deleted = util.asbool( deleted )
|
||||
q = kwd.get( 'q', '' )
|
||||
if q:
|
||||
page = kwd.get( 'page', 1 )
|
||||
return_jsonp = util.asbool( kwd.get( 'jsonp', False ) )
|
||||
callback = kwd.get( 'callback', 'callback' )
|
||||
search_results = self._search( trans, q, page )
|
||||
if return_jsonp:
|
||||
response = str( '%s(%s);' % ( callback, json.dumps( search_results ) ) )
|
||||
else:
|
||||
response = json.dumps( search_results )
|
||||
return response
|
||||
|
||||
clause_list = [ and_( trans.app.model.Repository.table.c.deprecated == False,
|
||||
trans.app.model.Repository.table.c.deleted == deleted ) ]
|
||||
if owner is not None:
|
||||
@@ -345,7 +382,28 @@ class RepositoriesController( BaseAPIController ):
|
||||
repository_dict[ 'category_ids' ] = \
|
||||
[ trans.security.encode_id( x.category.id ) for x in repository.categories ]
|
||||
repository_dicts.append( repository_dict )
|
||||
return repository_dicts
|
||||
return json.dumps( repository_dicts )
|
||||
|
||||
def _search( self, trans, q, page=1 ):
|
||||
"""
|
||||
Perform the search over TS repositories.
|
||||
Note that search works over the Whoosh index which you have
|
||||
to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually.
|
||||
Also TS config option toolshed_search_on has to be True and
|
||||
whoosh_index_dir has to be specified.
|
||||
"""
|
||||
if not self.app.config.toolshed_search_on:
|
||||
raise ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' )
|
||||
if not self.app.config.whoosh_index_dir:
|
||||
raise ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please contact the administrator.' )
|
||||
search_term = q.strip()
|
||||
if len( search_term ) < 3:
|
||||
raise RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' )
|
||||
|
||||
repo_search = RepoSearch()
|
||||
results = repo_search.search( trans, search_term, page )
|
||||
results[ 'hostname' ] = web.url_for( '/', qualified = True )
|
||||
return results
|
||||
|
||||
@web.expose_api
|
||||
def remove_repository_registry_entry( self, trans, payload, **kwd ):
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
"""API for searching the toolshed repositories"""
|
||||
from galaxy import exceptions
|
||||
from galaxy import eggs
|
||||
from galaxy import util
|
||||
from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
|
||||
from galaxy.web.base.controller import BaseAPIController
|
||||
from galaxy.webapps.tool_shed.search.repo_search import RepoSearch
|
||||
from galaxy.web import url_for
|
||||
import json
|
||||
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
|
||||
class SearchController ( BaseAPIController ):
|
||||
|
||||
@expose_api_raw_anonymous_and_sessionless
|
||||
def search( self, trans, search_term, **kwd ):
|
||||
"""
|
||||
Perform a search over the Whoosh index.
|
||||
The index has to be pre-created with build_ts_whoosh_index.sh.
|
||||
TS config option toolshed_search_on has to be turned on and
|
||||
whoosh_index_dir has to be specified.
|
||||
|
||||
:param search_term: (required)the term to search
|
||||
:type search_term: str
|
||||
|
||||
:param page: (optional)requested page of the search
|
||||
:type page: int
|
||||
|
||||
:param jsonp: (optional)flag whether to use jsonp format response, defaults to False
|
||||
:type jsonp: bool
|
||||
|
||||
:param callback: (optional)name of the function to wrap callback in
|
||||
used only when jsonp is true, defaults to 'callback'
|
||||
:type callback: str
|
||||
|
||||
:returns dict: object containing list of results and a hostname
|
||||
"""
|
||||
if not self.app.config.toolshed_search_on:
|
||||
raise exceptions.ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' )
|
||||
if not self.app.config.whoosh_index_dir:
|
||||
raise exceptions.ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please ontact the administrator.' )
|
||||
search_term = search_term.strip()
|
||||
if len( search_term ) < 3:
|
||||
raise exceptions.RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' )
|
||||
|
||||
page = kwd.get( 'page', 1 )
|
||||
return_jsonp = util.asbool( kwd.get( 'jsonp', False ) )
|
||||
callback = kwd.get( 'callback', 'callback' )
|
||||
|
||||
repo_search = RepoSearch()
|
||||
results = repo_search.search( trans, search_term, page )
|
||||
results[ 'hostname' ] = url_for( '/', qualified = True )
|
||||
|
||||
if return_jsonp:
|
||||
response = str( '%s(%s);' % ( callback, json.dumps( results ) ) )
|
||||
else:
|
||||
response = json.dumps( results )
|
||||
return response
|
||||
@@ -101,11 +101,6 @@ def app_factory( global_conf, **kwargs ):
|
||||
controller='groups',
|
||||
action='show',
|
||||
conditions=dict( method=[ "GET" ] ) )
|
||||
webapp.mapper.connect( 'repo_search',
|
||||
'/api/search/',
|
||||
controller='search',
|
||||
action='search',
|
||||
conditions=dict( method=[ "GET" ] ) )
|
||||
webapp.mapper.resource( 'category',
|
||||
'categories',
|
||||
controller='categories',
|
||||
|
||||
@@ -6,36 +6,26 @@ from galaxy.webapps.tool_shed import model
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
# Whoosh is compatible with Python 2.5+
|
||||
# Try to import Whoosh and set flag to indicate whether
|
||||
# the tool search is ready.
|
||||
try:
|
||||
eggs.require( "Whoosh" )
|
||||
import whoosh.index
|
||||
from whoosh import scoring
|
||||
from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED
|
||||
from whoosh.scoring import BM25F
|
||||
from whoosh.qparser import MultifieldParser
|
||||
from whoosh.index import Index
|
||||
search_ready = True
|
||||
eggs.require( "Whoosh" )
|
||||
import whoosh.index
|
||||
from whoosh import scoring
|
||||
from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED
|
||||
from whoosh.scoring import BM25F
|
||||
from whoosh.qparser import MultifieldParser
|
||||
from whoosh.index import Index
|
||||
|
||||
schema = Schema(
|
||||
id = STORED,
|
||||
name = TEXT( field_boost = 1.7, stored = True ),
|
||||
description = TEXT( field_boost = 1.5, stored = True ),
|
||||
long_description = TEXT( stored = True ),
|
||||
homepage_url = TEXT( stored = True ),
|
||||
remote_repository_url = TEXT( stored = True ),
|
||||
repo_owner_username = TEXT( stored = True ),
|
||||
times_downloaded = STORED,
|
||||
approved = STORED,
|
||||
last_updated = STORED,
|
||||
full_last_updated = STORED )
|
||||
|
||||
except ImportError, e:
|
||||
log.error( 'There was an error while importing the Whoosh library.' )
|
||||
search_ready = False
|
||||
schema = None
|
||||
schema = Schema(
|
||||
id = STORED,
|
||||
name = TEXT( field_boost = 1.7, stored = True ),
|
||||
description = TEXT( field_boost = 1.5, stored = True ),
|
||||
long_description = TEXT( stored = True ),
|
||||
homepage_url = TEXT( stored = True ),
|
||||
remote_repository_url = TEXT( stored = True ),
|
||||
repo_owner_username = TEXT( stored = True ),
|
||||
times_downloaded = STORED,
|
||||
approved = STORED,
|
||||
last_updated = STORED,
|
||||
full_last_updated = STORED )
|
||||
|
||||
|
||||
class RepoWeighting( scoring.BM25F ):
|
||||
@@ -82,60 +72,57 @@ class RepoSearch( object ):
|
||||
|
||||
:returns results: dictionary containing number of hits, hits themselves and matched terms for each
|
||||
"""
|
||||
if search_ready:
|
||||
whoosh_index_dir = trans.app.config.whoosh_index_dir
|
||||
index_exists = whoosh.index.exists_in( whoosh_index_dir )
|
||||
if index_exists:
|
||||
index = whoosh.index.open_dir( whoosh_index_dir )
|
||||
try:
|
||||
# Some literature about BM25F:
|
||||
# http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf
|
||||
# http://en.wikipedia.org/wiki/Okapi_BM25
|
||||
# __Basically__ the higher number the bigger weight.
|
||||
repo_weighting = RepoWeighting( field_B = { 'name_B' : 0.9,
|
||||
'description_B' : 0.6,
|
||||
'long_description_B' : 0.5,
|
||||
'homepage_url_B' : 0.3,
|
||||
'remote_repository_url_B' : 0.2,
|
||||
'repo_owner_username' : 0.3 } )
|
||||
whoosh_index_dir = trans.app.config.whoosh_index_dir
|
||||
index_exists = whoosh.index.exists_in( whoosh_index_dir )
|
||||
if index_exists:
|
||||
index = whoosh.index.open_dir( whoosh_index_dir )
|
||||
try:
|
||||
# Some literature about BM25F:
|
||||
# http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf
|
||||
# http://en.wikipedia.org/wiki/Okapi_BM25
|
||||
# __Basically__ the higher number the bigger weight.
|
||||
repo_weighting = RepoWeighting( field_B = { 'name_B' : 0.9,
|
||||
'description_B' : 0.6,
|
||||
'long_description_B' : 0.5,
|
||||
'homepage_url_B' : 0.3,
|
||||
'remote_repository_url_B' : 0.2,
|
||||
'repo_owner_username' : 0.3 } )
|
||||
|
||||
searcher = index.searcher( weighting = repo_weighting )
|
||||
searcher = index.searcher( weighting = repo_weighting )
|
||||
|
||||
parser = MultifieldParser( [
|
||||
'name',
|
||||
'description',
|
||||
'long_description',
|
||||
'homepage_url',
|
||||
'remote_repository_url',
|
||||
'repo_owner_username' ], schema = schema )
|
||||
parser = MultifieldParser( [
|
||||
'name',
|
||||
'description',
|
||||
'long_description',
|
||||
'homepage_url',
|
||||
'remote_repository_url',
|
||||
'repo_owner_username' ], schema = schema )
|
||||
|
||||
user_query = parser.parse( '*' + search_term + '*' )
|
||||
hits = searcher.search_page( user_query, page, pagelen = 10, terms = True )
|
||||
user_query = parser.parse( '*' + search_term + '*' )
|
||||
hits = searcher.search_page( user_query, page, pagelen = 10, terms = True )
|
||||
|
||||
log.debug( 'searching for: #' + str( search_term ) )
|
||||
log.debug( 'total hits: ' + str( len( hits ) ) )
|
||||
log.debug( 'scored hits: ' + str( hits.scored_length() ) )
|
||||
results = {}
|
||||
results[ 'total_results'] = str( len( hits ) )
|
||||
results[ 'hits' ] = []
|
||||
for hit in hits:
|
||||
hit_dict = {}
|
||||
hit_dict[ 'id' ] = trans.security.encode_id( hit.get( 'id' ) )
|
||||
hit_dict[ 'repo_owner_username' ] = hit.get( 'repo_owner_username' )
|
||||
hit_dict[ 'name' ] = hit.get( 'name' )
|
||||
hit_dict[ 'long_description' ] = hit.get( 'long_description' )
|
||||
hit_dict[ 'remote_repository_url' ] = hit.get( 'remote_repository_url' )
|
||||
hit_dict[ 'homepage_url' ] = hit.get( 'homepage_url' )
|
||||
hit_dict[ 'description' ] = hit.get( 'description' )
|
||||
hit_dict[ 'last_updated' ] = hit.get( 'last_updated' )
|
||||
hit_dict[ 'full_last_updated' ] = hit.get( 'full_last_updated' )
|
||||
hit_dict[ 'approved' ] = hit.get( 'approved' )
|
||||
hit_dict[ 'times_downloaded' ] = hit.get( 'times_downloaded' )
|
||||
results[ 'hits' ].append( {'repository': hit_dict, 'matched_terms': hit.matched_terms(), 'score': hit.score } )
|
||||
return results
|
||||
finally:
|
||||
searcher.close()
|
||||
else:
|
||||
raise exceptions.InternalServerError( 'The search index file is missing.' )
|
||||
log.debug( 'searching for: #' + str( search_term ) )
|
||||
log.debug( 'total hits: ' + str( len( hits ) ) )
|
||||
log.debug( 'scored hits: ' + str( hits.scored_length() ) )
|
||||
results = {}
|
||||
results[ 'total_results'] = str( len( hits ) )
|
||||
results[ 'hits' ] = []
|
||||
for hit in hits:
|
||||
hit_dict = {}
|
||||
hit_dict[ 'id' ] = trans.security.encode_id( hit.get( 'id' ) )
|
||||
hit_dict[ 'repo_owner_username' ] = hit.get( 'repo_owner_username' )
|
||||
hit_dict[ 'name' ] = hit.get( 'name' )
|
||||
hit_dict[ 'long_description' ] = hit.get( 'long_description' )
|
||||
hit_dict[ 'remote_repository_url' ] = hit.get( 'remote_repository_url' )
|
||||
hit_dict[ 'homepage_url' ] = hit.get( 'homepage_url' )
|
||||
hit_dict[ 'description' ] = hit.get( 'description' )
|
||||
hit_dict[ 'last_updated' ] = hit.get( 'last_updated' )
|
||||
hit_dict[ 'full_last_updated' ] = hit.get( 'full_last_updated' )
|
||||
hit_dict[ 'approved' ] = hit.get( 'approved' )
|
||||
hit_dict[ 'times_downloaded' ] = hit.get( 'times_downloaded' )
|
||||
results[ 'hits' ].append( {'repository': hit_dict, 'matched_terms': hit.matched_terms(), 'score': hit.score } )
|
||||
return results
|
||||
finally:
|
||||
searcher.close()
|
||||
else:
|
||||
raise exceptions.InternalServerError( 'Could not initialize search.' )
|
||||
raise exceptions.InternalServerError( 'The search index file is missing.' )
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
Build indexes for searching the TS.
|
||||
Run this script from the Tool Shed folder, example:
|
||||
|
||||
$ python scripts/tool_shed/build_ts_whoosh_index_test.py -c config/tool_shed.ini -r database/community_files
|
||||
$ python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.ini
|
||||
|
||||
Make sure you adjusted your config to:
|
||||
* turn on searching via toolshed_search_on
|
||||
* specify whoosh_index_dir where the indexes will be placed
|
||||
|
||||
Also make sure that GALAXY_EGGS_PATH variable is properly set.
|
||||
Also make sure that GALAXY_EGGS_PATH variable is properly set
|
||||
in case you are using non-default location for Galaxy.
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
@@ -205,10 +206,9 @@ def get_sa_session_and_needed_config_settings( path_to_tool_shed_config ):
|
||||
if __name__ == "__main__":
|
||||
parser = OptionParser()
|
||||
parser.add_option("-c", "--config", dest="path_to_tool_shed_config", default="config/tool_shed.ini", help="specify tool_shed.ini location")
|
||||
parser.add_option("-r", "--repos", dest="path_to_repositories", default="database/community_files", help="specify path to folder with repositories")
|
||||
(options, args) = parser.parse_args()
|
||||
path_to_tool_shed_config = options.path_to_tool_shed_config
|
||||
path_to_repositories = options.path_to_repositories
|
||||
sa_session, config_settings = get_sa_session_and_needed_config_settings( path_to_tool_shed_config )
|
||||
whoosh_index_dir = config_settings.get( 'whoosh_index_dir', None )
|
||||
path_to_repositories = config_settings.get( 'file_path', 'database/community_files' )
|
||||
build_index( sa_session, whoosh_index_dir, path_to_repositories )
|
||||
|
||||
Reference in New Issue
Block a user