diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index c92070157ef..fd0b33925b6 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -16,13 +16,16 @@ from galaxy.exceptions import InsufficientPermissionsException from galaxy.exceptions import ActionInputError from galaxy.exceptions import ObjectNotFound from galaxy.exceptions import MalformedId +from galaxy.exceptions import ConfigDoesNotAllowException from galaxy.datatypes import checkers from galaxy.model.orm import and_ from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless +from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless from galaxy.web.base.controller import BaseAPIController from galaxy.web.base.controller import HTTPBadRequest from galaxy.web.framework.helpers import time_ago +from galaxy.webapps.tool_shed.search.repo_search import RepoSearch from tool_shed.capsule import capsule_manager from tool_shed.metadata import repository_metadata_manager @@ -316,20 +319,54 @@ class RepositoriesController( BaseAPIController ): return_dict[ key ] = val return return_dict - @web.expose_api_anonymous + @expose_api_raw_anonymous_and_sessionless def index( self, trans, deleted=False, owner=None, name=None, **kwd ): """ GET /api/repositories + Displays a collection of repositories with optional criteria. - :param deleted: True/False, displays repositories that are or are not set to deleted. - :param owner: the owner's public username. - :param name: the repository name. + :param q: (optional)if present search on the given query will be performed + :type q: str - Displays a collection (list) of repositories. + :param page: (optional)requested page of the search + :type page: int + + :param jsonp: (optional)flag whether to use jsonp format response, defaults to False + :type jsonp: bool + + :param callback: (optional)name of the function to wrap callback in + used only when jsonp is true, defaults to 'callback' + :type callback: str + + :param deleted: (optional)displays repositories that are or are not set to deleted. + :type deleted: bool + + :param owner: (optional)the owner's public username. + :type owner: str + + :param name: (optional)the repository name. + :type name: str + + :returns dict: object containing list of results + + Examples: + GET http://localhost:9009/api/repositories + GET http://localhost:9009/api/repositories?q=fastq """ - # Example URL: http://localhost:9009/api/repositories repository_dicts = [] deleted = util.asbool( deleted ) + q = kwd.get( 'q', '' ) + if q: + page = kwd.get( 'page', 1 ) + return_jsonp = util.asbool( kwd.get( 'jsonp', False ) ) + callback = kwd.get( 'callback', 'callback' ) + search_results = self._search( trans, q, page ) + if return_jsonp: + response = str( '%s(%s);' % ( callback, json.dumps( search_results ) ) ) + else: + response = json.dumps( search_results ) + return response + clause_list = [ and_( trans.app.model.Repository.table.c.deprecated == False, trans.app.model.Repository.table.c.deleted == deleted ) ] if owner is not None: @@ -345,7 +382,28 @@ class RepositoriesController( BaseAPIController ): repository_dict[ 'category_ids' ] = \ [ trans.security.encode_id( x.category.id ) for x in repository.categories ] repository_dicts.append( repository_dict ) - return repository_dicts + return json.dumps( repository_dicts ) + + def _search( self, trans, q, page=1 ): + """ + Perform the search over TS repositories. + Note that search works over the Whoosh index which you have + to pre-create with scripts/tool_shed/build_ts_whoosh_index.sh manually. + Also TS config option toolshed_search_on has to be True and + whoosh_index_dir has to be specified. + """ + if not self.app.config.toolshed_search_on: + raise ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' ) + if not self.app.config.whoosh_index_dir: + raise ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please contact the administrator.' ) + search_term = q.strip() + if len( search_term ) < 3: + raise RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' ) + + repo_search = RepoSearch() + results = repo_search.search( trans, search_term, page ) + results[ 'hostname' ] = web.url_for( '/', qualified = True ) + return results @web.expose_api def remove_repository_registry_entry( self, trans, payload, **kwd ): diff --git a/lib/galaxy/webapps/tool_shed/api/search.py b/lib/galaxy/webapps/tool_shed/api/search.py deleted file mode 100644 index a1a1090058b..00000000000 --- a/lib/galaxy/webapps/tool_shed/api/search.py +++ /dev/null @@ -1,60 +0,0 @@ -"""API for searching the toolshed repositories""" -from galaxy import exceptions -from galaxy import eggs -from galaxy import util -from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless -from galaxy.web.base.controller import BaseAPIController -from galaxy.webapps.tool_shed.search.repo_search import RepoSearch -from galaxy.web import url_for -import json - -import logging -log = logging.getLogger( __name__ ) - - -class SearchController ( BaseAPIController ): - - @expose_api_raw_anonymous_and_sessionless - def search( self, trans, search_term, **kwd ): - """ - Perform a search over the Whoosh index. - The index has to be pre-created with build_ts_whoosh_index.sh. - TS config option toolshed_search_on has to be turned on and - whoosh_index_dir has to be specified. - - :param search_term: (required)the term to search - :type search_term: str - - :param page: (optional)requested page of the search - :type page: int - - :param jsonp: (optional)flag whether to use jsonp format response, defaults to False - :type jsonp: bool - - :param callback: (optional)name of the function to wrap callback in - used only when jsonp is true, defaults to 'callback' - :type callback: str - - :returns dict: object containing list of results and a hostname - """ - if not self.app.config.toolshed_search_on: - raise exceptions.ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' ) - if not self.app.config.whoosh_index_dir: - raise exceptions.ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please ontact the administrator.' ) - search_term = search_term.strip() - if len( search_term ) < 3: - raise exceptions.RequestParameterInvalidException( 'The search term has to be at least 3 characters long.' ) - - page = kwd.get( 'page', 1 ) - return_jsonp = util.asbool( kwd.get( 'jsonp', False ) ) - callback = kwd.get( 'callback', 'callback' ) - - repo_search = RepoSearch() - results = repo_search.search( trans, search_term, page ) - results[ 'hostname' ] = url_for( '/', qualified = True ) - - if return_jsonp: - response = str( '%s(%s);' % ( callback, json.dumps( results ) ) ) - else: - response = json.dumps( results ) - return response diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 7fa54a6166c..ef42eb7d821 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -101,11 +101,6 @@ def app_factory( global_conf, **kwargs ): controller='groups', action='show', conditions=dict( method=[ "GET" ] ) ) - webapp.mapper.connect( 'repo_search', - '/api/search/', - controller='search', - action='search', - conditions=dict( method=[ "GET" ] ) ) webapp.mapper.resource( 'category', 'categories', controller='categories', diff --git a/lib/galaxy/webapps/tool_shed/search/repo_search.py b/lib/galaxy/webapps/tool_shed/search/repo_search.py index 6c13a76119e..1af111a1028 100644 --- a/lib/galaxy/webapps/tool_shed/search/repo_search.py +++ b/lib/galaxy/webapps/tool_shed/search/repo_search.py @@ -6,36 +6,26 @@ from galaxy.webapps.tool_shed import model import logging log = logging.getLogger( __name__ ) -# Whoosh is compatible with Python 2.5+ -# Try to import Whoosh and set flag to indicate whether -# the tool search is ready. -try: - eggs.require( "Whoosh" ) - import whoosh.index - from whoosh import scoring - from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED - from whoosh.scoring import BM25F - from whoosh.qparser import MultifieldParser - from whoosh.index import Index - search_ready = True +eggs.require( "Whoosh" ) +import whoosh.index +from whoosh import scoring +from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED +from whoosh.scoring import BM25F +from whoosh.qparser import MultifieldParser +from whoosh.index import Index - schema = Schema( - id = STORED, - name = TEXT( field_boost = 1.7, stored = True ), - description = TEXT( field_boost = 1.5, stored = True ), - long_description = TEXT( stored = True ), - homepage_url = TEXT( stored = True ), - remote_repository_url = TEXT( stored = True ), - repo_owner_username = TEXT( stored = True ), - times_downloaded = STORED, - approved = STORED, - last_updated = STORED, - full_last_updated = STORED ) - -except ImportError, e: - log.error( 'There was an error while importing the Whoosh library.' ) - search_ready = False - schema = None +schema = Schema( + id = STORED, + name = TEXT( field_boost = 1.7, stored = True ), + description = TEXT( field_boost = 1.5, stored = True ), + long_description = TEXT( stored = True ), + homepage_url = TEXT( stored = True ), + remote_repository_url = TEXT( stored = True ), + repo_owner_username = TEXT( stored = True ), + times_downloaded = STORED, + approved = STORED, + last_updated = STORED, + full_last_updated = STORED ) class RepoWeighting( scoring.BM25F ): @@ -82,60 +72,57 @@ class RepoSearch( object ): :returns results: dictionary containing number of hits, hits themselves and matched terms for each """ - if search_ready: - whoosh_index_dir = trans.app.config.whoosh_index_dir - index_exists = whoosh.index.exists_in( whoosh_index_dir ) - if index_exists: - index = whoosh.index.open_dir( whoosh_index_dir ) - try: - # Some literature about BM25F: - # http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf - # http://en.wikipedia.org/wiki/Okapi_BM25 - # __Basically__ the higher number the bigger weight. - repo_weighting = RepoWeighting( field_B = { 'name_B' : 0.9, - 'description_B' : 0.6, - 'long_description_B' : 0.5, - 'homepage_url_B' : 0.3, - 'remote_repository_url_B' : 0.2, - 'repo_owner_username' : 0.3 } ) + whoosh_index_dir = trans.app.config.whoosh_index_dir + index_exists = whoosh.index.exists_in( whoosh_index_dir ) + if index_exists: + index = whoosh.index.open_dir( whoosh_index_dir ) + try: + # Some literature about BM25F: + # http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf + # http://en.wikipedia.org/wiki/Okapi_BM25 + # __Basically__ the higher number the bigger weight. + repo_weighting = RepoWeighting( field_B = { 'name_B' : 0.9, + 'description_B' : 0.6, + 'long_description_B' : 0.5, + 'homepage_url_B' : 0.3, + 'remote_repository_url_B' : 0.2, + 'repo_owner_username' : 0.3 } ) - searcher = index.searcher( weighting = repo_weighting ) + searcher = index.searcher( weighting = repo_weighting ) - parser = MultifieldParser( [ - 'name', - 'description', - 'long_description', - 'homepage_url', - 'remote_repository_url', - 'repo_owner_username' ], schema = schema ) + parser = MultifieldParser( [ + 'name', + 'description', + 'long_description', + 'homepage_url', + 'remote_repository_url', + 'repo_owner_username' ], schema = schema ) - user_query = parser.parse( '*' + search_term + '*' ) - hits = searcher.search_page( user_query, page, pagelen = 10, terms = True ) + user_query = parser.parse( '*' + search_term + '*' ) + hits = searcher.search_page( user_query, page, pagelen = 10, terms = True ) - log.debug( 'searching for: #' + str( search_term ) ) - log.debug( 'total hits: ' + str( len( hits ) ) ) - log.debug( 'scored hits: ' + str( hits.scored_length() ) ) - results = {} - results[ 'total_results'] = str( len( hits ) ) - results[ 'hits' ] = [] - for hit in hits: - hit_dict = {} - hit_dict[ 'id' ] = trans.security.encode_id( hit.get( 'id' ) ) - hit_dict[ 'repo_owner_username' ] = hit.get( 'repo_owner_username' ) - hit_dict[ 'name' ] = hit.get( 'name' ) - hit_dict[ 'long_description' ] = hit.get( 'long_description' ) - hit_dict[ 'remote_repository_url' ] = hit.get( 'remote_repository_url' ) - hit_dict[ 'homepage_url' ] = hit.get( 'homepage_url' ) - hit_dict[ 'description' ] = hit.get( 'description' ) - hit_dict[ 'last_updated' ] = hit.get( 'last_updated' ) - hit_dict[ 'full_last_updated' ] = hit.get( 'full_last_updated' ) - hit_dict[ 'approved' ] = hit.get( 'approved' ) - hit_dict[ 'times_downloaded' ] = hit.get( 'times_downloaded' ) - results[ 'hits' ].append( {'repository': hit_dict, 'matched_terms': hit.matched_terms(), 'score': hit.score } ) - return results - finally: - searcher.close() - else: - raise exceptions.InternalServerError( 'The search index file is missing.' ) + log.debug( 'searching for: #' + str( search_term ) ) + log.debug( 'total hits: ' + str( len( hits ) ) ) + log.debug( 'scored hits: ' + str( hits.scored_length() ) ) + results = {} + results[ 'total_results'] = str( len( hits ) ) + results[ 'hits' ] = [] + for hit in hits: + hit_dict = {} + hit_dict[ 'id' ] = trans.security.encode_id( hit.get( 'id' ) ) + hit_dict[ 'repo_owner_username' ] = hit.get( 'repo_owner_username' ) + hit_dict[ 'name' ] = hit.get( 'name' ) + hit_dict[ 'long_description' ] = hit.get( 'long_description' ) + hit_dict[ 'remote_repository_url' ] = hit.get( 'remote_repository_url' ) + hit_dict[ 'homepage_url' ] = hit.get( 'homepage_url' ) + hit_dict[ 'description' ] = hit.get( 'description' ) + hit_dict[ 'last_updated' ] = hit.get( 'last_updated' ) + hit_dict[ 'full_last_updated' ] = hit.get( 'full_last_updated' ) + hit_dict[ 'approved' ] = hit.get( 'approved' ) + hit_dict[ 'times_downloaded' ] = hit.get( 'times_downloaded' ) + results[ 'hits' ].append( {'repository': hit_dict, 'matched_terms': hit.matched_terms(), 'score': hit.score } ) + return results + finally: + searcher.close() else: - raise exceptions.InternalServerError( 'Could not initialize search.' ) + raise exceptions.InternalServerError( 'The search index file is missing.' ) diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index d23d175a435..1ee63caae51 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -2,13 +2,14 @@ Build indexes for searching the TS. Run this script from the Tool Shed folder, example: -$ python scripts/tool_shed/build_ts_whoosh_index_test.py -c config/tool_shed.ini -r database/community_files +$ python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.ini Make sure you adjusted your config to: * turn on searching via toolshed_search_on * specify whoosh_index_dir where the indexes will be placed -Also make sure that GALAXY_EGGS_PATH variable is properly set. +Also make sure that GALAXY_EGGS_PATH variable is properly set +in case you are using non-default location for Galaxy. """ import sys import os @@ -205,10 +206,9 @@ def get_sa_session_and_needed_config_settings( path_to_tool_shed_config ): if __name__ == "__main__": parser = OptionParser() parser.add_option("-c", "--config", dest="path_to_tool_shed_config", default="config/tool_shed.ini", help="specify tool_shed.ini location") - parser.add_option("-r", "--repos", dest="path_to_repositories", default="database/community_files", help="specify path to folder with repositories") (options, args) = parser.parse_args() path_to_tool_shed_config = options.path_to_tool_shed_config - path_to_repositories = options.path_to_repositories sa_session, config_settings = get_sa_session_and_needed_config_settings( path_to_tool_shed_config ) whoosh_index_dir = config_settings.get( 'whoosh_index_dir', None ) + path_to_repositories = config_settings.get( 'file_path', 'database/community_files' ) build_index( sa_session, whoosh_index_dir, path_to_repositories )